Wed, 31 Jan 2018 22:32:11 +0000
Added pause feature
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
1 | # |
89
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
2 | # Borgend by Tuomo Valkonen, 2018 |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
3 | # |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
4 | # This file implements the scheduling, running, and borg output processing |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
5 | # for a specific configured backup. |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
6 | # |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
7 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
8 | import logging |
5 | 9 | import time |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
10 | import datetime |
97 | 11 | import re |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
12 | from enum import IntEnum |
49 | 13 | from threading import Thread, Lock, Condition |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
14 | |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
15 | from . import config |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
16 | from . import repository |
97 | 17 | from .dreamtime import MonotonicTime, DreamTime, RealTime |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
18 | from .instance import BorgInstance |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
19 | from .scheduler import TerminableThread |
2 | 20 | |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
85
diff
changeset
|
21 | _logger=logging.getLogger(__name__) |
31 | 22 | |
98 | 23 | JOIN_TIMEOUT=10 |
64 | 24 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
25 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
26 | # State and operation related helper classes |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
27 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
28 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
29 | class State(IntEnum): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
30 | # State |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
31 | INACTIVE=0 |
98 | 32 | PAUSED=1 |
33 | SCHEDULED=2 | |
34 | QUEUED=3 | |
35 | ACTIVE=4 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
36 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
37 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
38 | class Errors(IntEnum): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
39 | OK=0 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
40 | BUSY=1 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
41 | OFFLINE=2 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
42 | ERRORS=3 |
45
aa2a95dc6093
Further improvements to state reporting; indicate busyness if repository lock cannot be acquired
Tuomo Valkonen <tuomov@iki.fi>
parents:
38
diff
changeset
|
43 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
44 | def combine(self, other): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
45 | return max(self, other) |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
46 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
47 | def ok(self): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
48 | return self==self.OK |
45
aa2a95dc6093
Further improvements to state reporting; indicate busyness if repository lock cannot be acquired
Tuomo Valkonen <tuomov@iki.fi>
parents:
38
diff
changeset
|
49 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
50 | def __str__(self): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
51 | return _errorstring[self] |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
52 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
53 | _errorstring={ |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
54 | Errors.OK: 'ok', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
55 | Errors.BUSY: 'busy', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
56 | Errors.OFFLINE: 'offline', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
57 | Errors.ERRORS: 'errors' |
6 | 58 | } |
59 | ||
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
60 | class Operation: |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
61 | CREATE='create' |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
62 | PRUNE='prune' |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
63 | INFO='info' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
64 | LIST='list' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
65 | |
97 | 66 | def __init__(self, type, start_time, **kwargs): |
67 | self.type=type | |
68 | self.start_time=start_time | |
69 | self.finish_time=None | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
70 | self.detail=kwargs |
97 | 71 | self.errors=Errors.OK |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
72 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
73 | def when(self): |
97 | 74 | return self.start_time.realtime() |
75 | ||
76 | def ok(self): | |
77 | return self.errors.ok() | |
78 | ||
79 | def add_error(self, error): | |
80 | self.errors=self.errors.combine(error) | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
81 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
82 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
83 | class Status(Operation): |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
84 | def __init__(self, backup, op=None): |
97 | 85 | op=None |
86 | errorsop=None | |
87 | ||
88 | if backup.current_operation: | |
89 | op=backup.current_operation | |
90 | elif backup.scheduled_operation: | |
91 | op=backup.scheduled_operation | |
92 | if backup.previous_operation: | |
93 | errorsop=backup.previous_operation | |
94 | ||
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
95 | if op: |
97 | 96 | super().__init__(op.type, op.start_time, **op.detail) |
97 | if not errorsop: | |
98 | errorsop=op | |
99 | self.errors=errorsop.errors | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
100 | else: |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
101 | super().__init__(None, None) |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
102 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
103 | self.name=backup.name |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
104 | self.state=backup.state |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
105 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
106 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
107 | # Miscellaneous helper routines |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
108 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
109 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
110 | loglevel_translation={ |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
111 | 'CRITICAL': logging.CRITICAL, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
112 | 'ERROR': logging.ERROR, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
113 | 'WARNING': logging.WARNING, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
114 | 'DEBUG': logging.DEBUG, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
115 | 'INFO': logging.INFO |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
116 | } |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
117 | |
6 | 118 | def translate_loglevel(x): |
119 | if x in loglevel_translation: | |
120 | return loglevel_translation[x] | |
121 | else: | |
122 | return logging.ERROR | |
123 | ||
15 | 124 | def safe_get_int(t, x): |
125 | if x in t: | |
126 | tmp=t[x] | |
127 | if isinstance(tmp, int): | |
128 | return tmp | |
129 | return None | |
130 | ||
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
131 | def parse_borg_date(d, logger): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
132 | try: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
133 | res=datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S.%f') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
134 | except Exception: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
135 | logger.exception('Unable parse date from borg: "%s"' % d) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
136 | res=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
137 | return res |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
138 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
139 | _checkpoint_str='.checkpoint' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
140 | _checkpoint_str_len=len(_checkpoint_str) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
141 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
142 | def is_checkpoint(name): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
143 | i=name.rfind(_checkpoint_str); |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
144 | return i>=0 and i+_checkpoint_str_len==len(name) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
145 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
146 | def get_archive_time(a, logger): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
147 | if not 'name' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
148 | logger.error('Borg archive list entry missing name') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
149 | return None, False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
150 | if is_checkpoint(a['name']): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
151 | logger.debug('Skipping checkpoint in archive list') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
152 | return None, True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
153 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
154 | thistime=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
155 | if 'start' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
156 | thistime=parse_borg_date(a['start'], logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
157 | if not thistime and 'time' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
158 | thistime=parse_borg_date(a['time'], logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
159 | if not thistime: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
160 | return None, False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
161 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
162 | return thistime, True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
163 | |
97 | 164 | _prune_progress_re=re.compile(".*\(([0-9]+)/([0-9]+)\)$") |
165 | # Borg gives very little progress info in easy form, so try to extrat it | |
166 | def check_prune_status(msg): | |
167 | res=_prune_progress_re.match(msg) | |
168 | if res: | |
169 | c=res.groups() | |
170 | try: | |
171 | archive_no=int(c[0]) | |
172 | of_total=int(c[1]) | |
173 | except: | |
174 | pass | |
175 | else: | |
176 | return archive_no, of_total | |
177 | return None, None | |
178 | ||
179 | ||
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
180 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
181 | # The Backup class |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
182 | # |
15 | 183 | |
49 | 184 | class Backup(TerminableThread): |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
185 | |
2 | 186 | def __decode_config(self, cfg): |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
187 | loc0='Backup %d' % self.identifier |
2 | 188 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
189 | self.backup_name=config.check_string(cfg, 'name', 'Name', loc0) |
2 | 190 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
191 | _logger.debug("Configuring backup '%s'" % self.backup_name) |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
192 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
193 | self.logger=_logger.getChild(self.backup_name) |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
194 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
195 | loc="Backup '%s'" % self.backup_name |
2 | 196 | |
54 | 197 | reponame=config.check_string(cfg, 'repository', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
198 | 'Target repository', loc) |
54 | 199 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
200 | self.repository=repository.find_repository(reponame) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
201 | if not self.repository: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
202 | raise Exception("Repository '%s' not configured" % reponame) |
2 | 203 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
204 | self.archive_prefix=config.check_string(cfg, 'archive_prefix', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
205 | 'Archive prefix', loc) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
206 | |
2 | 207 | self.archive_template=config.check_string(cfg, 'archive_template', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
208 | 'Archive template', loc) |
2 | 209 | |
210 | self.backup_interval=config.check_nonneg_int(cfg, 'backup_interval', | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
211 | 'Backup interval', loc, |
2 | 212 | config.defaults['backup_interval']) |
213 | ||
97 | 214 | self.prune_interval=config.check_nonneg_int(cfg, 'prune_interval', |
215 | 'Prune interval', loc, | |
216 | config.defaults['prune_interval']) | |
217 | ||
2 | 218 | self.retry_interval=config.check_nonneg_int(cfg, 'retry_interval', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
219 | 'Retry interval', loc, |
2 | 220 | config.defaults['retry_interval']) |
221 | ||
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
222 | |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
223 | scheduling=config.check_string(cfg, 'scheduling', |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
224 | 'Scheduling mode', loc, |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
225 | default="dreamtime") |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
226 | |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
227 | if scheduling=="dreamtime": |
97 | 228 | self.timeclass=DreamTime |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
229 | elif scheduling=="realtime": |
97 | 230 | self.timeclass=MonotonicTime |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
231 | elif scheduling=="manual": |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
232 | self.backup_interval=0 |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
233 | else: |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
234 | logging.error("Invalid time class '%s' for %s" % (scheduling, loc)) |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
235 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
236 | self.paths=config.check_nonempty_list_of_strings(cfg, 'paths', 'Paths', loc) |
32 | 237 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
238 | self.borg_parameters=config.BorgParameters.from_config(cfg, loc) |
20
fdfbe5d7b677
Keychain support and random fixes
Tuomo Valkonen <tuomov@iki.fi>
parents:
17
diff
changeset
|
239 | |
2 | 240 | |
49 | 241 | def __init__(self, identifier, cfg, scheduler): |
2 | 242 | self.identifier=identifier |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
243 | self.__status_update_callback=None |
98 | 244 | self._pause=False |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
245 | self.scheduler=scheduler |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
246 | self.logger=None # setup up in __decode_config once backup name is known |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
247 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
248 | self.borg_instance=None |
7 | 249 | self.thread_log=None |
8 | 250 | self.thread_res=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
251 | |
97 | 252 | self.previous_operation=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
253 | self.current_operation=None |
10 | 254 | self.scheduled_operation=None |
97 | 255 | self.previous_operation_of_type={} |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
256 | self.state=State.INACTIVE |
97 | 257 | self.timeclass=DreamTime |
49 | 258 | |
259 | self.__decode_config(cfg) | |
260 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
261 | super().__init__(target = self.__main_thread, name = self.backup_name) |
49 | 262 | self.daemon=True |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
263 | |
7 | 264 | def is_running(self): |
49 | 265 | with self._cond: |
8 | 266 | running=self.__is_running_unlocked() |
267 | return running | |
268 | ||
269 | def __is_running_unlocked(self): | |
270 | running=self.current_operation | |
271 | if not running: | |
272 | # Consistency check | |
273 | assert((not self.borg_instance and not self.thread_log and | |
274 | not self.thread_res)) | |
7 | 275 | return running |
276 | ||
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
277 | def __block_when_running(self): |
7 | 278 | running=self.is_running() |
279 | assert(not running) | |
2 | 280 | |
7 | 281 | def __log_listener(self): |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
282 | self.logger.debug('Log listener thread waiting for entries') |
7 | 283 | success=True |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
284 | for msg in iter(self.borg_instance.read_log, None): |
97 | 285 | self.logger.debug(str(msg)) |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
286 | t=msg['type'] |
15 | 287 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
288 | errormsg=None |
15 | 289 | callback=None |
290 | ||
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
291 | if t=='progress_percent': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
292 | current=safe_get_int(msg, 'current') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
293 | total=safe_get_int(msg, 'total') |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
294 | operation_no=safe_get_int(msg, 'operation') |
15 | 295 | if current is not None and total is not None: |
49 | 296 | with self._cond: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
297 | self.current_operation.detail['progress_current']=current |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
298 | self.current_operation.detail['progress_total']=total |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
299 | self.current_operation.detail['operation_no']=operation_no |
14
5a988a2c2624
UI: progress percentange support (Borg doesn't seem to be reporting) + error indicator in tray: B?
Tuomo Valkonen <tuomov@iki.fi>
parents:
12
diff
changeset
|
300 | status, callback=self.__status_unlocked() |
15 | 301 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
302 | elif t=='archive_progress': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
303 | original_size=safe_get_int(msg, 'original_size') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
304 | compressed_size=safe_get_int(msg, 'compressed_size') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
305 | deduplicated_size=safe_get_int(msg, 'deduplicated_size') |
15 | 306 | if original_size is not None and original_size is not None and deduplicated_size is not None: |
49 | 307 | with self._cond: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
308 | self.current_operation.detail['original_size']=original_size |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
309 | self.current_operation.detail['compressed_size']=compressed_size |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
310 | self.current_operation.detail['deduplicated_size']=deduplicated_size |
15 | 311 | status, callback=self.__status_unlocked() |
312 | ||
313 | elif t=='progress_message': | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
314 | pass |
15 | 315 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
316 | elif t=='file_status': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
317 | pass |
15 | 318 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
319 | elif t=='log_message': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
320 | if 'levelname' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
321 | msg['levelname']='ERROR' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
322 | if 'message' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
323 | msg['message']='UNKNOWN' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
324 | if 'name' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
325 | msg['name']='borg' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
326 | lvl=translate_loglevel(msg['levelname']) |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
327 | self.logger.log(lvl, msg['name'] + ': ' + msg['message']) |
71
a8a5ebb64e02
Changed retry timing to start form end of previous attempt instead of beginning
Tuomo Valkonen <tuomov@iki.fi>
parents:
70
diff
changeset
|
328 | if lvl>=logging.ERROR: |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
329 | errormsg=msg |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
330 | errors=Errors.ERRORS |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
331 | if ('msgid' in msg and |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
332 | (msg['msgid']=='LockTimeout' or # observed in reality |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
333 | msg['msgid']=='LockErrorT' or # in docs |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
334 | msg['msgid']=='LockErrorT')): # in docs |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
335 | errors=Errors.BUSY |
49 | 336 | with self._cond: |
97 | 337 | self.current_operation.add_error(errors) |
98 | 338 | # Don't notify of errors if we are terminating or pausing |
339 | if not self._terminate_or_pause(): | |
340 | status, callback=self.__status_unlocked() | |
97 | 341 | elif lvl==logging.INFO and self.current_operation.type==Operation.PRUNE: |
342 | # Borg gives very little progress info in easy form, so try to extrat it | |
343 | archive_number, of_total=check_prune_status(msg['message']) | |
344 | if archive_number!=None and of_total!=None: | |
345 | self.current_operation.detail['progress_current_secondary']=archive_number | |
346 | self.current_operation.detail['progress_total_secondary']=of_total | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
347 | status, callback=self.__status_unlocked() |
88
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
348 | |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
349 | elif t=='question_prompt' or t=='question_prompt_retry': |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
350 | self.logger.error('Did not expect to receive question prompt from borg') |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
351 | with self._cond: |
97 | 352 | self.current_operation.add_error(Errors.ERRORS) |
88
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
353 | # TODO: terminate org? Send 'NO' reply? |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
354 | |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
355 | elif (t=='question_invalid_answer' or t=='question_accepted_default' |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
356 | or t=='question_accepted_true' or t=='question_accepted_false' |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
357 | or t=='question_env_answer'): |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
358 | pass |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
359 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
360 | else: |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
361 | self.logger.debug('Unrecognised log entry %s' % str(status)) |
15 | 362 | |
363 | if callback: | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
364 | callback(status, errorlog=errormsg) |
6 | 365 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
366 | self.logger.debug('Waiting for borg subprocess to terminate in log thread') |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
367 | |
6 | 368 | self.borg_instance.wait() |
369 | ||
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
370 | self.logger.debug('Borg subprocess terminated; terminating log listener thread') |
7 | 371 | |
372 | def __result_listener(self): | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
373 | self.logger.debug('Result listener thread waiting for result') |
7 | 374 | |
375 | res=self.borg_instance.read_result() | |
376 | ||
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
377 | self.logger.debug('Borg result: %s' % str(res)) |
7 | 378 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
379 | if res is None: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
380 | with self._cond: |
97 | 381 | # Prune gives absolutely no result, so don't complain |
382 | if (self.current_operation.ok() and | |
383 | self.current_operation.type!=Operation.PRUNE): | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
384 | self.logger.error('No result from borg despite no error in log') |
97 | 385 | self.current_operation.add_error(Errors.ERRORS) |
386 | elif self.current_operation.type==Operation.LIST: | |
387 | self.__handle_list_result(res) | |
388 | ||
389 | # All other results are discarded | |
390 | ||
391 | def __handle_list_result(self, res): | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
392 | ok=True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
393 | latest=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
394 | if 'archives' in res and isinstance(res['archives'], list): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
395 | archives=res['archives'] |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
396 | for a in archives: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
397 | if not isinstance(a, dict): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
398 | self.logger.error('Borg archive list entry not a dictionary') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
399 | ok=False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
400 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
401 | thistime, this_ok=get_archive_time(a, self.logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
402 | if thistime and (not latest or thistime>latest): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
403 | latest=thistime |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
404 | ok=ok and this_ok |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
405 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
406 | logger.error('Borg archive list missing "archives" entry') |
7 | 407 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
408 | if not ok: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
409 | with self._cond: |
97 | 410 | self.current_operation.add_error(Errors.ERRORS) |
88
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
411 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
412 | if latest: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
413 | self.logger.info('borg info: Previous backup was on %s' % latest.isoformat()) |
97 | 414 | when=MonotonicTime.from_realtime(time.mktime(latest.timetuple())) |
415 | op=Operation(Operation.CREATE, when, reason='listed') | |
416 | op.finish_time=when | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
417 | with self._cond: |
97 | 418 | self.previous_operation_of_type[Operation.CREATE]=op |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
419 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
420 | self.logger.info('borg info: Could not discover a previous backup') |
7 | 421 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
422 | def __do_launch(self, op, archive_or_repository, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
423 | common_params, op_params, paths=[]): |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
424 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
425 | self.logger.debug('Creating BorgInstance') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
426 | |
97 | 427 | inst=BorgInstance(op.type, archive_or_repository, |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
428 | common_params, op_params, paths) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
429 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
430 | self.logger.debug('Launching BorgInstance via repository') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
431 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
432 | # Only the Repository object has access to the passphrase |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
433 | self.repository.launch_borg_instance(inst) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
434 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
435 | self.logger.debug('Creating listener threads') |
31 | 436 | |
7 | 437 | t_log=Thread(target=self.__log_listener) |
438 | t_log.daemon=True | |
2 | 439 | |
7 | 440 | t_res=Thread(target=self.__result_listener) |
441 | t_res.daemon=True | |
442 | ||
443 | self.thread_log=t_log | |
444 | self.thread_res=t_res | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
445 | self.borg_instance=inst |
64 | 446 | self.current_operation=op |
447 | # Update scheduled time to real starting time to schedule | |
448 | # next run relative to this | |
97 | 449 | self.current_operation.start_time=MonotonicTime.now() |
64 | 450 | self.state=State.ACTIVE |
451 | # Reset error status when starting a new operation | |
452 | self.__update_status() | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
453 | |
7 | 454 | t_log.start() |
455 | t_res.start() | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
456 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
457 | |
49 | 458 | def __launch(self, op): |
97 | 459 | self.logger.debug("Launching '%s'" % str(op.type)) |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
460 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
461 | params=(config.borg_parameters |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
462 | +self.repository.borg_parameters |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
463 | +self.borg_parameters) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
464 | |
97 | 465 | if op.type==Operation.CREATE: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
466 | archive="%s::%s%s" % (self.repository.location, |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
467 | self.archive_prefix, |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
468 | self.archive_template) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
469 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
470 | self.__do_launch(op, archive, params.common, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
471 | params.create, self.paths) |
97 | 472 | elif op.type==Operation.PRUNE: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
473 | self.__do_launch(op, self.repository.location, params.common, |
97 | 474 | [{'prefix': self.archive_prefix}] + params.prune) |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
475 | |
97 | 476 | elif op.type==Operation.LIST: |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
477 | self.__do_launch(op, self.repository.location, params.common, |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
478 | [{'prefix': self.archive_prefix}]) |
8 | 479 | else: |
97 | 480 | raise NotImplementedError("Invalid operation '%s'" % str(op.type)) |
2 | 481 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
482 | # This must be called with self._cond held. |
64 | 483 | def __launch_and_wait(self): |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
484 | op=self.scheduled_operation |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
485 | if not op: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
486 | self.logger.debug("Queued operation aborted") |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
487 | else: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
488 | self.scheduled_operation=None |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
489 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
490 | self.__launch(op) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
491 | |
64 | 492 | self.__wait_finish() |
493 | ||
98 | 494 | def _terminate_or_pause(self): |
495 | return self._terminate or self._pause | |
496 | ||
64 | 497 | def __wait_finish(self): |
97 | 498 | current=self.current_operation |
499 | ||
64 | 500 | # Wait for main logger thread to terminate, or for us to be terminated |
98 | 501 | while not self._terminate_or_pause() and self.thread_res.is_alive(): |
64 | 502 | self._cond.release() |
503 | self.thread_res.join(JOIN_TIMEOUT) | |
504 | self._cond.acquire() | |
505 | ||
98 | 506 | # If terminate or pause has been signalled, let outer termination handler |
64 | 507 | # take care of things (Within this Backup class, it would be cleanest |
508 | # to raise an exception instead, but in most other places it's better | |
509 | # to just check self._terminate, so we don't complicate things with | |
510 | # an extra exception.) | |
98 | 511 | if self.thread_res.is_alive(): |
64 | 512 | return |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
513 | |
64 | 514 | self.logger.debug('Waiting for borg and log subprocesses to terminate') |
515 | ||
516 | self._cond.release() | |
517 | self.thread_log.join() | |
518 | self._cond.acquire() | |
519 | ||
520 | if not self.borg_instance.wait(): | |
521 | self.logger.error('Borg subprocess did not terminate') | |
97 | 522 | curent.add_error(Errors.ERRORS) |
523 | ||
524 | current.finish_time=MonotonicTime.now() | |
64 | 525 | |
97 | 526 | self.previous_operation_of_type[current.type]=current |
527 | self.previous_operation=current | |
528 | self.current_operation=None | |
64 | 529 | self.thread_res=None |
530 | self.thread_log=None | |
531 | self.borg_instance=None | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
532 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
533 | def __main_thread(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
534 | with self._cond: |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
535 | while not self._terminate: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
536 | try: |
64 | 537 | assert(not self.current_operation) |
538 | self.__main_thread_wait_schedule() | |
98 | 539 | if (not self._terminate_or_pause() and self.scheduled_operation |
540 | and self.scheduled_operation.start_time <= MonotonicTime.now()): | |
64 | 541 | self.__main_thread_queue_and_launch() |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
542 | except Exception as err: |
98 | 543 | self.logger.exception("Exception in backup '%s'" % self.backup_name) |
544 | finally: | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
545 | self.__cleanup() |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
546 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
547 | def __cleanup(self): |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
548 | thread_log=self.thread_log |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
549 | thread_res=self.thread_res |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
550 | borg_instance=self.borg_instance |
98 | 551 | self.scheduled_operation=None |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
552 | self.thread_log=None |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
553 | self.thread_res=None |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
554 | self.borg_instance=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
555 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
556 | self._cond.release() |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
557 | try: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
558 | if borg_instance: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
559 | self.logger.debug("Terminating a borg instance") |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
560 | borg_instance.terminate() |
8 | 561 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
562 | if thread_log: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
563 | self.logger.debug("Waiting for log thread to terminate") |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
564 | thread_log.join() |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
565 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
566 | if thread_res: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
567 | self.logger.debug("Waiting for result thread to terminate") |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
568 | thread_res.join() |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
569 | finally: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
570 | self._cond.acquire() |
98 | 571 | self.current_operation=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
572 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
573 | # Main thread/2. Schedule next operation if there is no manually |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
574 | # requested one |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
575 | def __main_thread_wait_schedule(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
576 | op=None |
98 | 577 | if self._pause: |
578 | self.logger.info("Waiting for resume to be signalled") | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
579 | |
98 | 580 | self.state=State.PAUSED |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
581 | self.__update_status() |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
582 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
583 | self._cond.wait() |
98 | 584 | else: |
585 | if not self.scheduled_operation: | |
586 | op=self.__next_operation_unlocked() | |
587 | if op: | |
588 | self.logger.info("Scheduling '%s' (detail: %s) on %s [%s]" % | |
589 | (str(op.type), op.detail or 'none', | |
590 | op.start_time.isoformat(), | |
591 | op.start_time.__class__.__name__)) | |
592 | ||
593 | self.scheduled_operation=op | |
594 | self.state=State.SCHEDULED | |
595 | self.__update_status() | |
596 | ||
597 | # Wait under scheduled wait | |
598 | self.scheduler.wait_until(op.start_time, self._cond, self.backup_name) | |
599 | else: | |
600 | # Nothing scheduled - just wait | |
601 | self.logger.info("Waiting for manual scheduling") | |
602 | ||
603 | self.state=State.INACTIVE | |
604 | self.__update_status() | |
605 | ||
606 | self._cond.wait() | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
607 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
608 | # Main thread/3. If there is a scheduled operation (it might have been |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
609 | # changed manually from 'op' created in __main_thread_wait_schedule above), |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
610 | # queue it on the repository, and launch the operation once repository |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
611 | # available |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
612 | def __main_thread_queue_and_launch(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
613 | if self.scheduled_operation: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
614 | self.logger.debug("Queuing") |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
615 | self.state=State.QUEUED |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
616 | self.__update_status() |
64 | 617 | res=self.repository.queue_action(self._cond, |
618 | action=self.__launch_and_wait, | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
619 | name=self.backup_name) |
98 | 620 | if not res: |
64 | 621 | self.logger.debug("Queueing aborted") |
8 | 622 | |
623 | def __next_operation_unlocked(self): | |
97 | 624 | listop=self.__next_operation_list() |
625 | if listop: | |
626 | return listop | |
627 | ||
628 | create=self.__next_operation_type(Operation.CREATE, | |
629 | self.backup_interval, | |
630 | important=True, | |
631 | initial_reason='initial'); | |
632 | ||
633 | prune=self.__next_operation_type(Operation.PRUNE, | |
634 | self.prune_interval, | |
635 | important=False, | |
636 | initial_reason=None); | |
637 | ||
638 | if not prune: | |
639 | return create | |
640 | elif not create: | |
641 | return prune | |
642 | elif create.start_time < prune.start_time: | |
643 | return create | |
644 | else: | |
645 | return prune | |
646 | ||
647 | def __next_operation_list(self): | |
648 | reason='initial' | |
649 | # Unless manual backup has been chosen (backup_interval<=0), perform | |
650 | # repository listing if no previous create operation known, or if we | |
651 | # just pruned the repository | |
652 | if self.backup_interval<=0: | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
653 | return None |
97 | 654 | elif (self.previous_operation and |
655 | self.previous_operation.type==Operation.PRUNE): | |
656 | tm=MonotonicTime.now() | |
657 | reason='post-prune' | |
658 | elif Operation.LIST in self.previous_operation_of_type: | |
659 | prev=self.previous_operation_of_type[Operation.LIST] | |
660 | if prev.ok(): | |
661 | return None | |
662 | if self.retry_interval==0: | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
663 | # Do not retry in case of errors if retry interval is zero |
10 | 664 | return None |
97 | 665 | # Attempt after retry interval |
666 | tm=MonotonicTime.after_other(prev.finish_time, self.retry_interval) | |
667 | else: | |
668 | # Nothing has been attempted: run immediately | |
669 | tm=MonotonicTime.now() | |
670 | return Operation(Operation.LIST, tm, reason=reason) | |
671 | ||
672 | def __next_operation_type(self, optype, standard_interval, | |
673 | important=False, | |
674 | initial_reason=None): | |
675 | if optype not in self.previous_operation_of_type: | |
676 | # No previous operation exists; perform immediately | |
677 | # if important, otherwise after standard interval. | |
678 | # Do not perform if manual operation selected by | |
679 | # setting standard_interval<=0 | |
680 | if standard_interval<=0: | |
10 | 681 | return None |
682 | else: | |
97 | 683 | if important: |
684 | tm=MonotonicTime.now() | |
685 | else: | |
686 | tm=self.timeclass.after(standard_interval) | |
687 | if initial_reason: | |
688 | return Operation(optype, tm, reason=initial_reason) | |
96
de8ac6c470d8
Backup scheduling fixes in case of an initial backup
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
689 | else: |
97 | 690 | return Operation(optype, tm) |
5 | 691 | else: |
97 | 692 | # Previous operation has been performed; perform after |
693 | # retry interval if there were errors, otherwise after | |
694 | # standard interval. | |
695 | prev=self.previous_operation_of_type[optype] | |
696 | if not prev.ok(): | |
697 | tm=MonotonicTime.after_other(prev.start_time, | |
698 | self.retry_interval) | |
699 | return Operation(optype, tm, reason='retry') | |
700 | elif standard_interval>0: | |
701 | tm=self.timeclass.after_other(prev.start_time, | |
702 | standard_interval) | |
703 | return Operation(optype, tm) | |
704 | else: | |
705 | # Manual operation is standard_interval is zero. | |
706 | return None | |
10 | 707 | |
708 | def __status_unlocked(self): | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
709 | callback=self.__status_update_callback |
97 | 710 | status=Status(self) |
10 | 711 | |
712 | return status, callback | |
713 | ||
49 | 714 | def __update_status(self): |
715 | status, callback = self.__status_unlocked() | |
716 | if callback: | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
717 | #self._cond.release() |
49 | 718 | try: |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
719 | callback(status) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
720 | except Exception: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
721 | self.logger.exception("Status update error") |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
722 | #finally: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
723 | # self._cond.acquire() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
724 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
725 | # |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
726 | # Interface functions |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
727 | # |
49 | 728 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
729 | def set_status_update_callback(self, callback): |
49 | 730 | with self._cond: |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
731 | self.__status_update_callback=callback |
10 | 732 | |
49 | 733 | def status(self): |
734 | with self._cond: | |
735 | res=self.__status_unlocked() | |
736 | return res[0] | |
10 | 737 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
738 | def create(self): |
97 | 739 | op=Operation(Operation.CREATE, MonotonicTime.now(), reason='manual') |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
740 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
741 | self.scheduled_operation=op |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
742 | self._cond.notify() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
743 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
744 | def prune(self): |
97 | 745 | op=Operation(Operation.PRUNE, MonotonicTime.now(), reason='manual') |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
746 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
747 | self.scheduled_operation=op |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
748 | self._cond.notify() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
749 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
750 | # TODO: Decide exact (manual) abort mechanism. Perhaps two stages |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
751 | def abort(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
752 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
753 | if self.borg_instance: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
754 | self.borg_instance.terminate() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
755 | |
98 | 756 | def is_paused(self): |
757 | with self._cond: | |
758 | paused=self.state==State.PAUSED | |
759 | return paused | |
760 | ||
761 | def pause(self): | |
762 | with self._cond: | |
763 | self.logger.debug('Pause signalled') | |
764 | self.scheduled_operation=None | |
765 | self._pause=True | |
766 | self._cond.notify() | |
767 | ||
768 | def resume(self): | |
769 | with self._cond: | |
770 | self.logger.debug('Resume signalled') | |
771 | self._pause=False | |
772 | self._cond.notify() | |
773 |