Wed, 07 Feb 2018 20:39:01 +0000
Time snapshot fixes.
Python's default arguments are purely idiotic (aka. pythonic): generated
only once. This makes sense in a purely functional language, which Python
lightyears away from, but severely limits their usefulness in an imperative
language. Decorators also seem clumsy for this, as one would have to tell
the number of positional arguments for things to work nice, being able to
pass the snapshot both positionally and as keyword. No luck.
So have to do things the old-fashioned hard way.
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
1 | # |
89
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
2 | # Borgend by Tuomo Valkonen, 2018 |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
3 | # |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
4 | # This file implements the scheduling, running, and borg output processing |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
5 | # for a specific configured backup. |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
6 | # |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
7 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
8 | import logging |
5 | 9 | import time |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
10 | import datetime |
97 | 11 | import re |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
12 | from enum import IntEnum |
49 | 13 | from threading import Thread, Lock, Condition |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
14 | |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
15 | from . import config |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
16 | from . import repository |
97 | 17 | from .dreamtime import MonotonicTime, DreamTime, RealTime |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
18 | from .instance import BorgInstance |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
19 | from .scheduler import TerminableThread |
106
a7bdc239ef62
Added exeption protection decorators to callbacks.
Tuomo Valkonen <tuomov@iki.fi>
parents:
104
diff
changeset
|
20 | from .exprotect import protect_noreturn |
2 | 21 | |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
85
diff
changeset
|
22 | _logger=logging.getLogger(__name__) |
31 | 23 | |
98 | 24 | JOIN_TIMEOUT=10 |
64 | 25 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
26 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
27 | # State and operation related helper classes |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
28 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
29 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
30 | class State(IntEnum): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
31 | # State |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
32 | INACTIVE=0 |
98 | 33 | PAUSED=1 |
34 | SCHEDULED=2 | |
35 | QUEUED=3 | |
36 | ACTIVE=4 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
37 | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
38 | def __str__(self): |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
39 | return _statestring[self] |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
40 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
41 | class Errors(IntEnum): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
42 | OK=0 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
43 | BUSY=1 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
44 | OFFLINE=2 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
45 | ERRORS=3 |
45
aa2a95dc6093
Further improvements to state reporting; indicate busyness if repository lock cannot be acquired
Tuomo Valkonen <tuomov@iki.fi>
parents:
38
diff
changeset
|
46 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
47 | def combine(self, other): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
48 | return max(self, other) |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
49 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
50 | def ok(self): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
51 | return self==self.OK |
45
aa2a95dc6093
Further improvements to state reporting; indicate busyness if repository lock cannot be acquired
Tuomo Valkonen <tuomov@iki.fi>
parents:
38
diff
changeset
|
52 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
53 | def __str__(self): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
54 | return _errorstring[self] |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
55 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
56 | _errorstring={ |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
57 | Errors.OK: 'ok', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
58 | Errors.BUSY: 'busy', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
59 | Errors.OFFLINE: 'offline', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
60 | Errors.ERRORS: 'errors' |
6 | 61 | } |
62 | ||
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
63 | _statestring={ |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
64 | State.INACTIVE: 'inactive', |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
65 | State.PAUSED: 'paused', |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
66 | State.SCHEDULED: 'scheduled', |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
67 | State.QUEUED: 'queued', |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
68 | State.ACTIVE: 'active' |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
69 | } |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
70 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
71 | class Operation: |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
72 | CREATE='create' |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
73 | PRUNE='prune' |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
74 | INFO='info' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
75 | LIST='list' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
76 | |
97 | 77 | def __init__(self, type, start_time, **kwargs): |
78 | self.type=type | |
79 | self.start_time=start_time | |
80 | self.finish_time=None | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
81 | self.detail=kwargs |
97 | 82 | self.errors=Errors.OK |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
83 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
84 | def when(self): |
97 | 85 | return self.start_time.realtime() |
86 | ||
87 | def ok(self): | |
88 | return self.errors.ok() | |
89 | ||
90 | def add_error(self, error): | |
91 | self.errors=self.errors.combine(error) | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
92 | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
93 | def name(self): |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
94 | if 'reason' in self.detail: |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
95 | return str(self.type) + '.' + self.detail['reason'] |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
96 | else: |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
97 | return str(self.type) |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
98 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
99 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
100 | class Status(Operation): |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
101 | def __init__(self, backup, op=None): |
108
6409aa77ddd5
Error status wasn't correctly transmitted to UI after recent changes.
Tuomo Valkonen <tuomov@iki.fi>
parents:
107
diff
changeset
|
102 | op=backup.current_operation |
109
246190bfd501
Better error reporting logic.
Tuomo Valkonen <tuomov@iki.fi>
parents:
108
diff
changeset
|
103 | errorsop=backup.current_operation |
97 | 104 | |
108
6409aa77ddd5
Error status wasn't correctly transmitted to UI after recent changes.
Tuomo Valkonen <tuomov@iki.fi>
parents:
107
diff
changeset
|
105 | if not op: |
97 | 106 | op=backup.scheduled_operation |
107 | ||
109
246190bfd501
Better error reporting logic.
Tuomo Valkonen <tuomov@iki.fi>
parents:
108
diff
changeset
|
108 | if not errorsop: |
246190bfd501
Better error reporting logic.
Tuomo Valkonen <tuomov@iki.fi>
parents:
108
diff
changeset
|
109 | errorsop=backup.previous_operation |
246190bfd501
Better error reporting logic.
Tuomo Valkonen <tuomov@iki.fi>
parents:
108
diff
changeset
|
110 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
111 | if op: |
97 | 112 | super().__init__(op.type, op.start_time, **op.detail) |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
113 | else: |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
114 | super().__init__(None, None) |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
115 | |
108
6409aa77ddd5
Error status wasn't correctly transmitted to UI after recent changes.
Tuomo Valkonen <tuomov@iki.fi>
parents:
107
diff
changeset
|
116 | if errorsop: |
6409aa77ddd5
Error status wasn't correctly transmitted to UI after recent changes.
Tuomo Valkonen <tuomov@iki.fi>
parents:
107
diff
changeset
|
117 | self.errors=errorsop.errors |
6409aa77ddd5
Error status wasn't correctly transmitted to UI after recent changes.
Tuomo Valkonen <tuomov@iki.fi>
parents:
107
diff
changeset
|
118 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
119 | self.name=backup.name |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
120 | self.state=backup.state |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
121 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
122 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
123 | # Miscellaneous helper routines |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
124 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
125 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
126 | loglevel_translation={ |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
127 | 'CRITICAL': logging.CRITICAL, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
128 | 'ERROR': logging.ERROR, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
129 | 'WARNING': logging.WARNING, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
130 | 'DEBUG': logging.DEBUG, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
131 | 'INFO': logging.INFO |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
132 | } |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
133 | |
6 | 134 | def translate_loglevel(x): |
135 | if x in loglevel_translation: | |
136 | return loglevel_translation[x] | |
137 | else: | |
138 | return logging.ERROR | |
139 | ||
15 | 140 | def safe_get_int(t, x): |
141 | if x in t: | |
142 | tmp=t[x] | |
143 | if isinstance(tmp, int): | |
144 | return tmp | |
145 | return None | |
146 | ||
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
147 | def parse_borg_date(d, logger): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
148 | try: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
149 | res=datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S.%f') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
150 | except Exception: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
151 | logger.exception('Unable parse date from borg: "%s"' % d) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
152 | res=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
153 | return res |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
154 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
155 | _checkpoint_str='.checkpoint' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
156 | _checkpoint_str_len=len(_checkpoint_str) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
157 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
158 | def is_checkpoint(name): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
159 | i=name.rfind(_checkpoint_str); |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
160 | return i>=0 and i+_checkpoint_str_len==len(name) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
161 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
162 | def get_archive_time(a, logger): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
163 | if not 'name' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
164 | logger.error('Borg archive list entry missing name') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
165 | return None, False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
166 | if is_checkpoint(a['name']): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
167 | logger.debug('Skipping checkpoint in archive list') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
168 | return None, True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
169 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
170 | thistime=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
171 | if 'start' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
172 | thistime=parse_borg_date(a['start'], logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
173 | if not thistime and 'time' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
174 | thistime=parse_borg_date(a['time'], logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
175 | if not thistime: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
176 | return None, False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
177 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
178 | return thistime, True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
179 | |
97 | 180 | _prune_progress_re=re.compile(".*\(([0-9]+)/([0-9]+)\)$") |
181 | # Borg gives very little progress info in easy form, so try to extrat it | |
182 | def check_prune_status(msg): | |
183 | res=_prune_progress_re.match(msg) | |
184 | if res: | |
185 | c=res.groups() | |
186 | try: | |
187 | archive_no=int(c[0]) | |
188 | of_total=int(c[1]) | |
189 | except: | |
190 | pass | |
191 | else: | |
192 | return archive_no, of_total | |
193 | return None, None | |
194 | ||
195 | ||
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
196 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
197 | # The Backup class |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
198 | # |
15 | 199 | |
49 | 200 | class Backup(TerminableThread): |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
201 | |
2 | 202 | def __decode_config(self, cfg): |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
203 | loc0='Backup %d' % self.identifier |
2 | 204 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
205 | self.backup_name=config.check_string(cfg, 'name', 'Name', loc0) |
2 | 206 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
207 | _logger.debug("Configuring backup '%s'" % self.backup_name) |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
208 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
209 | self.logger=_logger.getChild(self.backup_name) |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
210 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
211 | loc="Backup '%s'" % self.backup_name |
2 | 212 | |
54 | 213 | reponame=config.check_string(cfg, 'repository', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
214 | 'Target repository', loc) |
54 | 215 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
216 | self.repository=repository.find_repository(reponame) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
217 | if not self.repository: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
218 | raise Exception("Repository '%s' not configured" % reponame) |
2 | 219 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
220 | self.archive_prefix=config.check_string(cfg, 'archive_prefix', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
221 | 'Archive prefix', loc) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
222 | |
2 | 223 | self.archive_template=config.check_string(cfg, 'archive_template', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
224 | 'Archive template', loc) |
2 | 225 | |
226 | self.backup_interval=config.check_nonneg_int(cfg, 'backup_interval', | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
227 | 'Backup interval', loc, |
2 | 228 | config.defaults['backup_interval']) |
229 | ||
97 | 230 | self.prune_interval=config.check_nonneg_int(cfg, 'prune_interval', |
231 | 'Prune interval', loc, | |
232 | config.defaults['prune_interval']) | |
233 | ||
2 | 234 | self.retry_interval=config.check_nonneg_int(cfg, 'retry_interval', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
235 | 'Retry interval', loc, |
2 | 236 | config.defaults['retry_interval']) |
237 | ||
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
238 | |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
239 | scheduling=config.check_string(cfg, 'scheduling', |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
240 | 'Scheduling mode', loc, |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
241 | default="dreamtime") |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
242 | |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
243 | if scheduling=="dreamtime": |
97 | 244 | self.timeclass=DreamTime |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
245 | elif scheduling=="realtime": |
97 | 246 | self.timeclass=MonotonicTime |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
247 | elif scheduling=="manual": |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
248 | self.backup_interval=0 |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
249 | else: |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
250 | logging.error("Invalid time class '%s' for %s" % (scheduling, loc)) |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
251 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
252 | self.paths=config.check_nonempty_list_of_strings(cfg, 'paths', 'Paths', loc) |
32 | 253 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
254 | self.borg_parameters=config.BorgParameters.from_config(cfg, loc) |
20
fdfbe5d7b677
Keychain support and random fixes
Tuomo Valkonen <tuomov@iki.fi>
parents:
17
diff
changeset
|
255 | |
2 | 256 | |
49 | 257 | def __init__(self, identifier, cfg, scheduler): |
2 | 258 | self.identifier=identifier |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
259 | self.__status_update_callback=None |
98 | 260 | self._pause=False |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
261 | self.scheduler=scheduler |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
262 | self.logger=None # setup up in __decode_config once backup name is known |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
263 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
264 | self.borg_instance=None |
7 | 265 | self.thread_log=None |
8 | 266 | self.thread_res=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
267 | |
97 | 268 | self.previous_operation=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
269 | self.current_operation=None |
10 | 270 | self.scheduled_operation=None |
97 | 271 | self.previous_operation_of_type={} |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
272 | self.state=State.INACTIVE |
97 | 273 | self.timeclass=DreamTime |
49 | 274 | |
275 | self.__decode_config(cfg) | |
276 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
277 | super().__init__(target = self.__main_thread, name = self.backup_name) |
49 | 278 | self.daemon=True |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
279 | |
7 | 280 | def is_running(self): |
49 | 281 | with self._cond: |
8 | 282 | running=self.__is_running_unlocked() |
283 | return running | |
284 | ||
285 | def __is_running_unlocked(self): | |
286 | running=self.current_operation | |
287 | if not running: | |
288 | # Consistency check | |
289 | assert((not self.borg_instance and not self.thread_log and | |
290 | not self.thread_res)) | |
7 | 291 | return running |
292 | ||
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
293 | def __block_when_running(self): |
7 | 294 | running=self.is_running() |
295 | assert(not running) | |
2 | 296 | |
106
a7bdc239ef62
Added exeption protection decorators to callbacks.
Tuomo Valkonen <tuomov@iki.fi>
parents:
104
diff
changeset
|
297 | @protect_noreturn |
7 | 298 | def __log_listener(self): |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
299 | self.logger.debug('Log listener thread waiting for entries') |
7 | 300 | success=True |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
301 | for msg in iter(self.borg_instance.read_log, None): |
97 | 302 | self.logger.debug(str(msg)) |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
303 | t=msg['type'] |
15 | 304 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
305 | errormsg=None |
15 | 306 | callback=None |
307 | ||
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
308 | if t=='progress_percent': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
309 | current=safe_get_int(msg, 'current') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
310 | total=safe_get_int(msg, 'total') |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
311 | operation_no=safe_get_int(msg, 'operation') |
15 | 312 | if current is not None and total is not None: |
49 | 313 | with self._cond: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
314 | self.current_operation.detail['progress_current']=current |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
315 | self.current_operation.detail['progress_total']=total |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
316 | self.current_operation.detail['operation_no']=operation_no |
14
5a988a2c2624
UI: progress percentange support (Borg doesn't seem to be reporting) + error indicator in tray: B?
Tuomo Valkonen <tuomov@iki.fi>
parents:
12
diff
changeset
|
317 | status, callback=self.__status_unlocked() |
15 | 318 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
319 | elif t=='archive_progress': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
320 | original_size=safe_get_int(msg, 'original_size') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
321 | compressed_size=safe_get_int(msg, 'compressed_size') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
322 | deduplicated_size=safe_get_int(msg, 'deduplicated_size') |
15 | 323 | if original_size is not None and original_size is not None and deduplicated_size is not None: |
49 | 324 | with self._cond: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
325 | self.current_operation.detail['original_size']=original_size |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
326 | self.current_operation.detail['compressed_size']=compressed_size |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
327 | self.current_operation.detail['deduplicated_size']=deduplicated_size |
15 | 328 | status, callback=self.__status_unlocked() |
329 | ||
330 | elif t=='progress_message': | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
331 | pass |
15 | 332 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
333 | elif t=='file_status': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
334 | pass |
15 | 335 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
336 | elif t=='log_message': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
337 | if 'levelname' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
338 | msg['levelname']='ERROR' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
339 | if 'message' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
340 | msg['message']='UNKNOWN' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
341 | if 'name' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
342 | msg['name']='borg' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
343 | lvl=translate_loglevel(msg['levelname']) |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
344 | self.logger.log(lvl, msg['name'] + ': ' + msg['message']) |
71
a8a5ebb64e02
Changed retry timing to start form end of previous attempt instead of beginning
Tuomo Valkonen <tuomov@iki.fi>
parents:
70
diff
changeset
|
345 | if lvl>=logging.ERROR: |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
346 | errormsg=msg |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
347 | errors=Errors.ERRORS |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
348 | if ('msgid' in msg and |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
349 | (msg['msgid']=='LockTimeout' or # observed in reality |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
350 | msg['msgid']=='LockErrorT' or # in docs |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
351 | msg['msgid']=='LockErrorT')): # in docs |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
352 | errors=Errors.BUSY |
49 | 353 | with self._cond: |
97 | 354 | self.current_operation.add_error(errors) |
98 | 355 | # Don't notify of errors if we are terminating or pausing |
356 | if not self._terminate_or_pause(): | |
357 | status, callback=self.__status_unlocked() | |
97 | 358 | elif lvl==logging.INFO and self.current_operation.type==Operation.PRUNE: |
359 | # Borg gives very little progress info in easy form, so try to extrat it | |
360 | archive_number, of_total=check_prune_status(msg['message']) | |
361 | if archive_number!=None and of_total!=None: | |
362 | self.current_operation.detail['progress_current_secondary']=archive_number | |
363 | self.current_operation.detail['progress_total_secondary']=of_total | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
364 | status, callback=self.__status_unlocked() |
88
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
365 | |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
366 | elif t=='question_prompt' or t=='question_prompt_retry': |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
367 | self.logger.error('Did not expect to receive question prompt from borg') |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
368 | with self._cond: |
97 | 369 | self.current_operation.add_error(Errors.ERRORS) |
88
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
370 | # TODO: terminate org? Send 'NO' reply? |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
371 | |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
372 | elif (t=='question_invalid_answer' or t=='question_accepted_default' |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
373 | or t=='question_accepted_true' or t=='question_accepted_false' |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
374 | or t=='question_env_answer'): |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
375 | pass |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
376 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
377 | else: |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
378 | self.logger.debug('Unrecognised log entry %s' % str(status)) |
15 | 379 | |
380 | if callback: | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
381 | callback(status, errorlog=errormsg) |
6 | 382 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
383 | self.logger.debug('Waiting for borg subprocess to terminate in log thread') |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
384 | |
6 | 385 | self.borg_instance.wait() |
386 | ||
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
387 | self.logger.debug('Borg subprocess terminated; terminating log listener thread') |
7 | 388 | |
106
a7bdc239ef62
Added exeption protection decorators to callbacks.
Tuomo Valkonen <tuomov@iki.fi>
parents:
104
diff
changeset
|
389 | @protect_noreturn |
7 | 390 | def __result_listener(self): |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
391 | self.logger.debug('Result listener thread waiting for result') |
7 | 392 | |
393 | res=self.borg_instance.read_result() | |
394 | ||
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
395 | self.logger.debug('Borg result: %s' % str(res)) |
7 | 396 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
397 | if res is None: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
398 | with self._cond: |
97 | 399 | # Prune gives absolutely no result, so don't complain |
400 | if (self.current_operation.ok() and | |
401 | self.current_operation.type!=Operation.PRUNE): | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
402 | self.logger.error('No result from borg despite no error in log') |
97 | 403 | self.current_operation.add_error(Errors.ERRORS) |
404 | elif self.current_operation.type==Operation.LIST: | |
405 | self.__handle_list_result(res) | |
406 | ||
407 | # All other results are discarded | |
408 | ||
409 | def __handle_list_result(self, res): | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
410 | ok=True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
411 | latest=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
412 | if 'archives' in res and isinstance(res['archives'], list): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
413 | archives=res['archives'] |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
414 | for a in archives: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
415 | if not isinstance(a, dict): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
416 | self.logger.error('Borg archive list entry not a dictionary') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
417 | ok=False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
418 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
419 | thistime, this_ok=get_archive_time(a, self.logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
420 | if thistime and (not latest or thistime>latest): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
421 | latest=thistime |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
422 | ok=ok and this_ok |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
423 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
424 | logger.error('Borg archive list missing "archives" entry') |
7 | 425 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
426 | if not ok: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
427 | with self._cond: |
97 | 428 | self.current_operation.add_error(Errors.ERRORS) |
88
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
429 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
430 | if latest: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
431 | self.logger.info('borg info: Previous backup was on %s' % latest.isoformat()) |
97 | 432 | when=MonotonicTime.from_realtime(time.mktime(latest.timetuple())) |
433 | op=Operation(Operation.CREATE, when, reason='listed') | |
434 | op.finish_time=when | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
435 | with self._cond: |
97 | 436 | self.previous_operation_of_type[Operation.CREATE]=op |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
437 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
438 | self.logger.info('borg info: Could not discover a previous backup') |
7 | 439 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
440 | def __do_launch(self, op, archive_or_repository, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
441 | common_params, op_params, paths=[]): |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
442 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
443 | self.logger.debug('Creating BorgInstance') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
444 | |
97 | 445 | inst=BorgInstance(op.type, archive_or_repository, |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
446 | common_params, op_params, paths) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
447 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
448 | self.logger.debug('Launching BorgInstance via repository') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
449 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
450 | # Only the Repository object has access to the passphrase |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
451 | self.repository.launch_borg_instance(inst) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
452 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
453 | self.logger.debug('Creating listener threads') |
31 | 454 | |
7 | 455 | t_log=Thread(target=self.__log_listener) |
456 | t_log.daemon=True | |
2 | 457 | |
7 | 458 | t_res=Thread(target=self.__result_listener) |
459 | t_res.daemon=True | |
460 | ||
461 | self.thread_log=t_log | |
462 | self.thread_res=t_res | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
463 | self.borg_instance=inst |
64 | 464 | self.current_operation=op |
465 | # Update scheduled time to real starting time to schedule | |
466 | # next run relative to this | |
97 | 467 | self.current_operation.start_time=MonotonicTime.now() |
64 | 468 | # Reset error status when starting a new operation |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
469 | self.__update_status(State.ACTIVE) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
470 | |
7 | 471 | t_log.start() |
472 | t_res.start() | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
473 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
474 | |
49 | 475 | def __launch(self, op): |
97 | 476 | self.logger.debug("Launching '%s'" % str(op.type)) |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
477 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
478 | params=(config.borg_parameters |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
479 | +self.repository.borg_parameters |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
480 | +self.borg_parameters) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
481 | |
97 | 482 | if op.type==Operation.CREATE: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
483 | archive="%s::%s%s" % (self.repository.location, |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
484 | self.archive_prefix, |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
485 | self.archive_template) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
486 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
487 | self.__do_launch(op, archive, params.common, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
488 | params.create, self.paths) |
97 | 489 | elif op.type==Operation.PRUNE: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
490 | self.__do_launch(op, self.repository.location, params.common, |
97 | 491 | [{'prefix': self.archive_prefix}] + params.prune) |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
492 | |
97 | 493 | elif op.type==Operation.LIST: |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
494 | self.__do_launch(op, self.repository.location, params.common, |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
495 | [{'prefix': self.archive_prefix}]) |
8 | 496 | else: |
97 | 497 | raise NotImplementedError("Invalid operation '%s'" % str(op.type)) |
2 | 498 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
499 | # This must be called with self._cond held. |
64 | 500 | def __launch_and_wait(self): |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
501 | op=self.scheduled_operation |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
502 | if not op: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
503 | self.logger.debug("Queued operation aborted") |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
504 | else: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
505 | self.scheduled_operation=None |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
506 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
507 | self.__launch(op) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
508 | |
64 | 509 | self.__wait_finish() |
510 | ||
98 | 511 | def _terminate_or_pause(self): |
512 | return self._terminate or self._pause | |
513 | ||
64 | 514 | def __wait_finish(self): |
97 | 515 | current=self.current_operation |
516 | ||
64 | 517 | # Wait for main logger thread to terminate, or for us to be terminated |
98 | 518 | while not self._terminate_or_pause() and self.thread_res.is_alive(): |
64 | 519 | self._cond.release() |
520 | self.thread_res.join(JOIN_TIMEOUT) | |
521 | self._cond.acquire() | |
522 | ||
98 | 523 | # If terminate or pause has been signalled, let outer termination handler |
64 | 524 | # take care of things (Within this Backup class, it would be cleanest |
525 | # to raise an exception instead, but in most other places it's better | |
526 | # to just check self._terminate, so we don't complicate things with | |
527 | # an extra exception.) | |
98 | 528 | if self.thread_res.is_alive(): |
64 | 529 | return |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
530 | |
64 | 531 | self.logger.debug('Waiting for borg and log subprocesses to terminate') |
532 | ||
533 | self._cond.release() | |
534 | self.thread_log.join() | |
535 | self._cond.acquire() | |
536 | ||
537 | if not self.borg_instance.wait(): | |
538 | self.logger.error('Borg subprocess did not terminate') | |
97 | 539 | curent.add_error(Errors.ERRORS) |
540 | ||
541 | current.finish_time=MonotonicTime.now() | |
64 | 542 | |
97 | 543 | self.previous_operation_of_type[current.type]=current |
544 | self.previous_operation=current | |
545 | self.current_operation=None | |
64 | 546 | self.thread_res=None |
547 | self.thread_log=None | |
548 | self.borg_instance=None | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
549 | |
106
a7bdc239ef62
Added exeption protection decorators to callbacks.
Tuomo Valkonen <tuomov@iki.fi>
parents:
104
diff
changeset
|
550 | @protect_noreturn |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
551 | def __main_thread(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
552 | with self._cond: |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
553 | while not self._terminate: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
554 | try: |
64 | 555 | assert(not self.current_operation) |
556 | self.__main_thread_wait_schedule() | |
100
b141bed9e718
macOS "sleep" signals / status are complete bollocks:
Tuomo Valkonen <tuomov@iki.fi>
parents:
98
diff
changeset
|
557 | if ((not self._terminate_or_pause()) and self.scheduled_operation |
98 | 558 | and self.scheduled_operation.start_time <= MonotonicTime.now()): |
64 | 559 | self.__main_thread_queue_and_launch() |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
560 | except Exception as err: |
98 | 561 | self.logger.exception("Exception in backup '%s'" % self.backup_name) |
562 | finally: | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
563 | self.__cleanup() |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
564 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
565 | def __cleanup(self): |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
566 | thread_log=self.thread_log |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
567 | thread_res=self.thread_res |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
568 | borg_instance=self.borg_instance |
98 | 569 | self.scheduled_operation=None |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
570 | self.thread_log=None |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
571 | self.thread_res=None |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
572 | self.borg_instance=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
573 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
574 | self._cond.release() |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
575 | try: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
576 | if borg_instance: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
577 | self.logger.debug("Terminating a borg instance") |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
578 | borg_instance.terminate() |
8 | 579 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
580 | if thread_log: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
581 | self.logger.debug("Waiting for log thread to terminate") |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
582 | thread_log.join() |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
583 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
584 | if thread_res: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
585 | self.logger.debug("Waiting for result thread to terminate") |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
586 | thread_res.join() |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
587 | finally: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
588 | self._cond.acquire() |
98 | 589 | self.current_operation=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
590 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
591 | # Main thread/2. Schedule next operation if there is no manually |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
592 | # requested one |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
593 | def __main_thread_wait_schedule(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
594 | op=None |
98 | 595 | if self._pause: |
596 | self.logger.info("Waiting for resume to be signalled") | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
597 | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
598 | self.__update_status(State.PAUSED) |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
599 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
600 | self._cond.wait() |
98 | 601 | else: |
602 | if not self.scheduled_operation: | |
603 | op=self.__next_operation_unlocked() | |
604 | if op: | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
605 | self.scheduled_operation=op |
98 | 606 | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
607 | self.__update_status(State.SCHEDULED) |
98 | 608 | |
609 | # Wait under scheduled wait | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
610 | eventname=op.name() + '@' + self.backup_name |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
611 | self.scheduler.wait_until(op.start_time, self._cond, eventname) |
98 | 612 | else: |
613 | # Nothing scheduled - just wait | |
614 | self.logger.info("Waiting for manual scheduling") | |
615 | ||
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
616 | self.__update_status(State.INACTIVE) |
98 | 617 | |
618 | self._cond.wait() | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
619 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
620 | # Main thread/3. If there is a scheduled operation (it might have been |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
621 | # changed manually from 'op' created in __main_thread_wait_schedule above), |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
622 | # queue it on the repository, and launch the operation once repository |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
623 | # available |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
624 | def __main_thread_queue_and_launch(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
625 | if self.scheduled_operation: |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
626 | |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
627 | self.__update_status(State.QUEUED) |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
628 | |
64 | 629 | res=self.repository.queue_action(self._cond, |
630 | action=self.__launch_and_wait, | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
631 | name=self.backup_name) |
98 | 632 | if not res: |
64 | 633 | self.logger.debug("Queueing aborted") |
8 | 634 | |
635 | def __next_operation_unlocked(self): | |
97 | 636 | listop=self.__next_operation_list() |
637 | if listop: | |
638 | return listop | |
639 | ||
640 | create=self.__next_operation_type(Operation.CREATE, | |
641 | self.backup_interval, | |
642 | important=True, | |
643 | initial_reason='initial'); | |
644 | ||
645 | prune=self.__next_operation_type(Operation.PRUNE, | |
646 | self.prune_interval, | |
647 | important=False, | |
648 | initial_reason=None); | |
649 | ||
650 | if not prune: | |
651 | return create | |
652 | elif not create: | |
653 | return prune | |
654 | elif create.start_time < prune.start_time: | |
655 | return create | |
656 | else: | |
657 | return prune | |
658 | ||
659 | def __next_operation_list(self): | |
660 | reason='initial' | |
661 | # Unless manual backup has been chosen (backup_interval<=0), perform | |
662 | # repository listing if no previous create operation known, or if we | |
663 | # just pruned the repository | |
664 | if self.backup_interval<=0: | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
665 | return None |
97 | 666 | elif (self.previous_operation and |
667 | self.previous_operation.type==Operation.PRUNE): | |
668 | tm=MonotonicTime.now() | |
669 | reason='post-prune' | |
670 | elif Operation.LIST in self.previous_operation_of_type: | |
671 | prev=self.previous_operation_of_type[Operation.LIST] | |
672 | if prev.ok(): | |
673 | return None | |
107
f63835e9f29e
Oops, a zero retry_interval check had gone missing in scheduling reorganisation.
Tuomo Valkonen <tuomov@iki.fi>
parents:
106
diff
changeset
|
674 | if self.retry_interval<=0: |
f63835e9f29e
Oops, a zero retry_interval check had gone missing in scheduling reorganisation.
Tuomo Valkonen <tuomov@iki.fi>
parents:
106
diff
changeset
|
675 | # Do not retry in case of errors if retry interval is <= 0 |
10 | 676 | return None |
97 | 677 | # Attempt after retry interval |
678 | tm=MonotonicTime.after_other(prev.finish_time, self.retry_interval) | |
679 | else: | |
680 | # Nothing has been attempted: run immediately | |
681 | tm=MonotonicTime.now() | |
682 | return Operation(Operation.LIST, tm, reason=reason) | |
683 | ||
684 | def __next_operation_type(self, optype, standard_interval, | |
685 | important=False, | |
686 | initial_reason=None): | |
687 | if optype not in self.previous_operation_of_type: | |
688 | # No previous operation exists; perform immediately | |
689 | # if important, otherwise after standard interval. | |
690 | # Do not perform if manual operation selected by | |
691 | # setting standard_interval<=0 | |
692 | if standard_interval<=0: | |
10 | 693 | return None |
694 | else: | |
97 | 695 | if important: |
696 | tm=MonotonicTime.now() | |
697 | else: | |
698 | tm=self.timeclass.after(standard_interval) | |
699 | if initial_reason: | |
700 | return Operation(optype, tm, reason=initial_reason) | |
96
de8ac6c470d8
Backup scheduling fixes in case of an initial backup
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
701 | else: |
97 | 702 | return Operation(optype, tm) |
5 | 703 | else: |
97 | 704 | # Previous operation has been performed; perform after |
705 | # retry interval if there were errors, otherwise after | |
706 | # standard interval. | |
707 | prev=self.previous_operation_of_type[optype] | |
708 | if not prev.ok(): | |
107
f63835e9f29e
Oops, a zero retry_interval check had gone missing in scheduling reorganisation.
Tuomo Valkonen <tuomov@iki.fi>
parents:
106
diff
changeset
|
709 | # Do not retry in case of errors if retry interval is <= 0 |
f63835e9f29e
Oops, a zero retry_interval check had gone missing in scheduling reorganisation.
Tuomo Valkonen <tuomov@iki.fi>
parents:
106
diff
changeset
|
710 | if self.retry_interval<=0: |
f63835e9f29e
Oops, a zero retry_interval check had gone missing in scheduling reorganisation.
Tuomo Valkonen <tuomov@iki.fi>
parents:
106
diff
changeset
|
711 | return None |
97 | 712 | tm=MonotonicTime.after_other(prev.start_time, |
713 | self.retry_interval) | |
714 | return Operation(optype, tm, reason='retry') | |
715 | elif standard_interval>0: | |
716 | tm=self.timeclass.after_other(prev.start_time, | |
717 | standard_interval) | |
718 | return Operation(optype, tm) | |
719 | else: | |
720 | # Manual operation is standard_interval is zero. | |
721 | return None | |
10 | 722 | |
723 | def __status_unlocked(self): | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
724 | callback=self.__status_update_callback |
97 | 725 | status=Status(self) |
10 | 726 | |
727 | return status, callback | |
728 | ||
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
729 | def __update_status(self, state): |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
730 | self.logger.debug("Entering %s state", str(state)) |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
731 | self.state=state |
49 | 732 | status, callback = self.__status_unlocked() |
733 | if callback: | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
734 | #self._cond.release() |
49 | 735 | try: |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
736 | callback(status) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
737 | except Exception: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
738 | self.logger.exception("Status update error") |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
739 | #finally: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
740 | # self._cond.acquire() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
741 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
742 | # |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
743 | # Interface functions |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
744 | # |
49 | 745 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
746 | def set_status_update_callback(self, callback): |
49 | 747 | with self._cond: |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
748 | self.__status_update_callback=callback |
10 | 749 | |
49 | 750 | def status(self): |
751 | with self._cond: | |
752 | res=self.__status_unlocked() | |
753 | return res[0] | |
10 | 754 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
755 | def create(self): |
97 | 756 | op=Operation(Operation.CREATE, MonotonicTime.now(), reason='manual') |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
757 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
758 | self.scheduled_operation=op |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
759 | self._cond.notify() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
760 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
761 | def prune(self): |
97 | 762 | op=Operation(Operation.PRUNE, MonotonicTime.now(), reason='manual') |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
763 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
764 | self.scheduled_operation=op |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
765 | self._cond.notify() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
766 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
767 | # TODO: Decide exact (manual) abort mechanism. Perhaps two stages |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
768 | def abort(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
769 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
770 | if self.borg_instance: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
771 | self.borg_instance.terminate() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
772 | |
98 | 773 | def is_paused(self): |
774 | with self._cond: | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
775 | paused=(self.state==State.PAUSED) |
98 | 776 | return paused |
777 | ||
778 | def pause(self): | |
779 | with self._cond: | |
780 | self.logger.debug('Pause signalled') | |
781 | self.scheduled_operation=None | |
782 | self._pause=True | |
783 | self._cond.notify() | |
784 | ||
785 | def resume(self): | |
786 | with self._cond: | |
787 | self.logger.debug('Resume signalled') | |
788 | self._pause=False | |
789 | self._cond.notify() | |
790 |