Sun, 28 Jan 2018 17:34:06 +0000
On startup, for better scheduling, obtain previous backup time with 'borg list'
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
1 | # |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
2 | # Borgend Backup instance |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
3 | # |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
4 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
5 | import logging |
5 | 6 | import time |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
7 | import datetime |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
8 | from enum import IntEnum |
49 | 9 | from threading import Thread, Lock, Condition |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
10 | |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
11 | from . import config |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
12 | from . import loggers |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
13 | from . import repository |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
14 | from . import dreamtime |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
15 | from .instance import BorgInstance |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
16 | from .scheduler import TerminableThread |
2 | 17 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
18 | _logger=loggers.get(__name__) |
31 | 19 | |
64 | 20 | JOIN_TIMEOUT=60 |
21 | ||
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
22 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
23 | # State and operation related helper classes |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
24 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
25 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
26 | class State(IntEnum): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
27 | # State |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
28 | INACTIVE=0 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
29 | SCHEDULED=1 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
30 | QUEUED=2 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
31 | ACTIVE=3 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
32 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
33 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
34 | class Errors(IntEnum): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
35 | OK=0 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
36 | BUSY=1 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
37 | OFFLINE=2 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
38 | ERRORS=3 |
45
aa2a95dc6093
Further improvements to state reporting; indicate busyness if repository lock cannot be acquired
Tuomo Valkonen <tuomov@iki.fi>
parents:
38
diff
changeset
|
39 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
40 | def combine(self, other): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
41 | return max(self, other) |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
42 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
43 | def ok(self): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
44 | return self==self.OK |
45
aa2a95dc6093
Further improvements to state reporting; indicate busyness if repository lock cannot be acquired
Tuomo Valkonen <tuomov@iki.fi>
parents:
38
diff
changeset
|
45 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
46 | def __str__(self): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
47 | return _errorstring[self] |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
48 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
49 | _errorstring={ |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
50 | Errors.OK: 'ok', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
51 | Errors.BUSY: 'busy', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
52 | Errors.OFFLINE: 'offline', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
53 | Errors.ERRORS: 'errors' |
6 | 54 | } |
55 | ||
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
56 | class Operation: |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
57 | CREATE='create' |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
58 | PRUNE='prune' |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
59 | INFO='info' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
60 | LIST='list' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
61 | |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
62 | def __init__(self, operation, time, **kwargs): |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
63 | self.operation=operation |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
64 | self.time=time |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
65 | self.detail=kwargs |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
66 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
67 | def when(self): |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
68 | return self.time.realtime() |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
69 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
70 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
71 | class Status(Operation): |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
72 | def __init__(self, backup, op=None): |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
73 | if op: |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
74 | super().__init__(op.operation, op.time, **op.detail) |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
75 | else: |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
76 | super().__init__(None, None) |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
77 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
78 | self.name=backup.name |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
79 | self.state=backup.state |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
80 | self.errors=backup.errors |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
81 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
82 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
83 | # Miscellaneous helper routines |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
84 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
85 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
86 | loglevel_translation={ |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
87 | 'CRITICAL': logging.CRITICAL, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
88 | 'ERROR': logging.ERROR, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
89 | 'WARNING': logging.WARNING, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
90 | 'DEBUG': logging.DEBUG, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
91 | 'INFO': logging.INFO |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
92 | } |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
93 | |
6 | 94 | def translate_loglevel(x): |
95 | if x in loglevel_translation: | |
96 | return loglevel_translation[x] | |
97 | else: | |
98 | return logging.ERROR | |
99 | ||
15 | 100 | def safe_get_int(t, x): |
101 | if x in t: | |
102 | tmp=t[x] | |
103 | if isinstance(tmp, int): | |
104 | return tmp | |
105 | return None | |
106 | ||
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
107 | def parse_borg_date(d, logger): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
108 | try: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
109 | res=datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S.%f') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
110 | except Exception: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
111 | logger.exception('Unable parse date from borg: "%s"' % d) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
112 | res=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
113 | return res |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
114 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
115 | _checkpoint_str='.checkpoint' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
116 | _checkpoint_str_len=len(_checkpoint_str) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
117 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
118 | def is_checkpoint(name): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
119 | i=name.rfind(_checkpoint_str); |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
120 | return i>=0 and i+_checkpoint_str_len==len(name) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
121 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
122 | def get_archive_time(a, logger): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
123 | if not 'name' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
124 | logger.error('Borg archive list entry missing name') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
125 | return None, False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
126 | if is_checkpoint(a['name']): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
127 | logger.debug('Skipping checkpoint in archive list') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
128 | return None, True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
129 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
130 | thistime=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
131 | if 'start' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
132 | thistime=parse_borg_date(a['start'], logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
133 | if not thistime and 'time' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
134 | thistime=parse_borg_date(a['time'], logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
135 | if not thistime: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
136 | return None, False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
137 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
138 | return thistime, True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
139 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
140 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
141 | # The Backup class |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
142 | # |
15 | 143 | |
49 | 144 | class Backup(TerminableThread): |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
145 | |
2 | 146 | def __decode_config(self, cfg): |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
147 | loc0='Backup %d' % self.identifier |
2 | 148 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
149 | self.backup_name=config.check_string(cfg, 'name', 'Name', loc0) |
2 | 150 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
151 | _logger.debug("Configuring backup '%s'" % self.backup_name) |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
152 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
153 | self.logger=_logger.getChild(self.backup_name) |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
154 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
155 | loc="Backup '%s'" % self.backup_name |
2 | 156 | |
54 | 157 | reponame=config.check_string(cfg, 'repository', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
158 | 'Target repository', loc) |
54 | 159 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
160 | self.repository=repository.find_repository(reponame) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
161 | if not self.repository: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
162 | raise Exception("Repository '%s' not configured" % reponame) |
2 | 163 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
164 | self.archive_prefix=config.check_string(cfg, 'archive_prefix', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
165 | 'Archive prefix', loc) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
166 | |
2 | 167 | self.archive_template=config.check_string(cfg, 'archive_template', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
168 | 'Archive template', loc) |
2 | 169 | |
170 | self.backup_interval=config.check_nonneg_int(cfg, 'backup_interval', | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
171 | 'Backup interval', loc, |
2 | 172 | config.defaults['backup_interval']) |
173 | ||
174 | self.retry_interval=config.check_nonneg_int(cfg, 'retry_interval', | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
175 | 'Retry interval', loc, |
2 | 176 | config.defaults['retry_interval']) |
177 | ||
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
178 | |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
179 | scheduling=config.check_string(cfg, 'scheduling', |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
180 | 'Scheduling mode', loc, |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
181 | default="dreamtime") |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
182 | |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
183 | if scheduling=="dreamtime": |
78
83b43987e61e
Renamed the "sleep" module "dreamtime"
Tuomo Valkonen <tuomov@iki.fi>
parents:
76
diff
changeset
|
184 | self.timeclass=dreamtime.DreamTime |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
185 | elif scheduling=="realtime": |
78
83b43987e61e
Renamed the "sleep" module "dreamtime"
Tuomo Valkonen <tuomov@iki.fi>
parents:
76
diff
changeset
|
186 | self.timeclass=dreamtime.MonotonicTime |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
187 | elif scheduling=="manual": |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
188 | self.backup_interval=0 |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
189 | else: |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
190 | logging.error("Invalid time class '%s' for %s" % (scheduling, loc)) |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
191 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
192 | self.paths=config.check_nonempty_list_of_strings(cfg, 'paths', 'Paths', loc) |
32 | 193 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
194 | self.borg_parameters=config.BorgParameters.from_config(cfg, loc) |
20
fdfbe5d7b677
Keychain support and random fixes
Tuomo Valkonen <tuomov@iki.fi>
parents:
17
diff
changeset
|
195 | |
2 | 196 | |
49 | 197 | def __init__(self, identifier, cfg, scheduler): |
2 | 198 | self.identifier=identifier |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
199 | self.__status_update_callback=None |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
200 | self.scheduler=scheduler |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
201 | self.logger=None # setup up in __decode_config once backup name is known |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
202 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
203 | self.borg_instance=None |
7 | 204 | self.thread_log=None |
8 | 205 | self.thread_res=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
206 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
207 | self.current_operation=None |
10 | 208 | self.scheduled_operation=None |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
209 | self.last_operation_finished=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
210 | self.last_create_when=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
211 | self.last_create_finished=None |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
212 | self.state=State.INACTIVE |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
213 | self.errors=Errors.OK |
78
83b43987e61e
Renamed the "sleep" module "dreamtime"
Tuomo Valkonen <tuomov@iki.fi>
parents:
76
diff
changeset
|
214 | self.timeclass=dreamtime.DreamTime |
49 | 215 | |
216 | self.__decode_config(cfg) | |
217 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
218 | super().__init__(target = self.__main_thread, name = self.backup_name) |
49 | 219 | self.daemon=True |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
220 | |
7 | 221 | def is_running(self): |
49 | 222 | with self._cond: |
8 | 223 | running=self.__is_running_unlocked() |
224 | return running | |
225 | ||
226 | def __is_running_unlocked(self): | |
227 | running=self.current_operation | |
228 | if not running: | |
229 | # Consistency check | |
230 | assert((not self.borg_instance and not self.thread_log and | |
231 | not self.thread_res)) | |
7 | 232 | return running |
233 | ||
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
234 | def __block_when_running(self): |
7 | 235 | running=self.is_running() |
236 | assert(not running) | |
2 | 237 | |
7 | 238 | def __log_listener(self): |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
239 | self.logger.debug('Log listener thread waiting for entries') |
7 | 240 | success=True |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
241 | for msg in iter(self.borg_instance.read_log, None): |
64 | 242 | self.logger.info(str(msg)) |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
243 | t=msg['type'] |
15 | 244 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
245 | errormsg=None |
15 | 246 | callback=None |
247 | ||
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
248 | if t=='progress_percent': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
249 | current=safe_get_int(msg, 'current') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
250 | total=safe_get_int(msg, 'total') |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
251 | operation_no=safe_get_int(msg, 'operation') |
15 | 252 | if current is not None and total is not None: |
49 | 253 | with self._cond: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
254 | self.current_operation.detail['progress_current']=current |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
255 | self.current_operation.detail['progress_total']=total |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
256 | self.current_operation.detail['operation_no']=operation_no |
14
5a988a2c2624
UI: progress percentange support (Borg doesn't seem to be reporting) + error indicator in tray: B?
Tuomo Valkonen <tuomov@iki.fi>
parents:
12
diff
changeset
|
257 | status, callback=self.__status_unlocked() |
15 | 258 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
259 | elif t=='archive_progress': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
260 | original_size=safe_get_int(msg, 'original_size') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
261 | compressed_size=safe_get_int(msg, 'compressed_size') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
262 | deduplicated_size=safe_get_int(msg, 'deduplicated_size') |
15 | 263 | if original_size is not None and original_size is not None and deduplicated_size is not None: |
49 | 264 | with self._cond: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
265 | self.current_operation.detail['original_size']=original_size |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
266 | self.current_operation.detail['compressed_size']=compressed_size |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
267 | self.current_operation.detail['deduplicated_size']=deduplicated_size |
15 | 268 | status, callback=self.__status_unlocked() |
269 | ||
270 | elif t=='progress_message': | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
271 | pass |
15 | 272 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
273 | elif t=='file_status': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
274 | pass |
15 | 275 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
276 | elif t=='log_message': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
277 | if 'levelname' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
278 | msg['levelname']='ERROR' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
279 | if 'message' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
280 | msg['message']='UNKNOWN' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
281 | if 'name' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
282 | msg['name']='borg' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
283 | lvl=translate_loglevel(msg['levelname']) |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
284 | self.logger.log(lvl, msg['name'] + ': ' + msg['message']) |
71
a8a5ebb64e02
Changed retry timing to start form end of previous attempt instead of beginning
Tuomo Valkonen <tuomov@iki.fi>
parents:
70
diff
changeset
|
285 | if lvl>=logging.ERROR: |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
286 | errormsg=msg |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
287 | errors=Errors.ERRORS |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
288 | if ('msgid' in msg and |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
289 | (msg['msgid']=='LockTimeout' or # observed in reality |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
290 | msg['msgid']=='LockErrorT' or # in docs |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
291 | msg['msgid']=='LockErrorT')): # in docs |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
292 | errors=Errors.BUSY |
49 | 293 | with self._cond: |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
294 | self.errors=self.errors.combine(errors) |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
295 | status, callback=self.__status_unlocked() |
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
296 | else: |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
297 | self.logger.debug('Unrecognised log entry %s' % str(status)) |
15 | 298 | |
299 | if callback: | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
300 | callback(status, errors=errormsg) |
6 | 301 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
302 | self.logger.debug('Waiting for borg subprocess to terminate in log thread') |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
303 | |
6 | 304 | self.borg_instance.wait() |
305 | ||
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
306 | self.logger.debug('Borg subprocess terminated; terminating log listener thread') |
7 | 307 | |
308 | def __result_listener(self): | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
309 | self.logger.debug('Result listener thread waiting for result') |
7 | 310 | |
311 | res=self.borg_instance.read_result() | |
312 | ||
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
313 | self.logger.debug('Borg result: %s' % str(res)) |
7 | 314 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
315 | if res is None: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
316 | with self._cond: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
317 | if self.errors.ok(): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
318 | self.logger.error('No result from borg despite no error in log') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
319 | self.errors=Errors.ERRORS |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
320 | elif self.current_operation.operation==Operation.CREATE: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
321 | with self._cond: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
322 | self.last_create_finished=time.monotonic() |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
323 | self.last_create_when=self.current_operation.time.monotonic() |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
324 | elif self.current_operation.operation==Operation.LIST: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
325 | ok=True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
326 | latest=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
327 | if 'archives' in res and isinstance(res['archives'], list): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
328 | archives=res['archives'] |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
329 | for a in archives: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
330 | if not isinstance(a, dict): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
331 | self.logger.error('Borg archive list entry not a dictionary') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
332 | ok=False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
333 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
334 | thistime, this_ok=get_archive_time(a, self.logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
335 | if thistime and (not latest or thistime>latest): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
336 | latest=thistime |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
337 | ok=ok and this_ok |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
338 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
339 | logger.error('Borg archive list missing "archives" entry') |
7 | 340 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
341 | if not ok: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
342 | with self._cond: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
343 | self.errors=self.errors.combine(Errors.ERRORS) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
344 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
345 | if latest: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
346 | self.logger.info('borg info: Previous backup was on %s' % latest.isoformat()) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
347 | realtime=time.mktime(latest.timetuple()) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
348 | monotonic=realtime+time.monotonic()-time.time() |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
349 | with self._cond: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
350 | self.last_create_finished=monotonic |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
351 | self.last_create_when=monotonic |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
352 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
353 | self.logger.info('borg info: Could not discover a previous backup') |
7 | 354 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
355 | def __do_launch(self, op, archive_or_repository, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
356 | common_params, op_params, paths=[]): |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
357 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
358 | inst=BorgInstance(op.operation, archive_or_repository, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
359 | common_params, op_params, paths) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
360 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
361 | # Only the Repository object has access to the passphrase |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
362 | self.repository.launch_borg_instance(inst) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
363 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
364 | self.logger.debug('Creating listener threads') |
31 | 365 | |
7 | 366 | t_log=Thread(target=self.__log_listener) |
367 | t_log.daemon=True | |
2 | 368 | |
7 | 369 | t_res=Thread(target=self.__result_listener) |
370 | t_res.daemon=True | |
371 | ||
372 | self.thread_log=t_log | |
373 | self.thread_res=t_res | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
374 | self.borg_instance=inst |
64 | 375 | self.current_operation=op |
376 | # Update scheduled time to real starting time to schedule | |
377 | # next run relative to this | |
78
83b43987e61e
Renamed the "sleep" module "dreamtime"
Tuomo Valkonen <tuomov@iki.fi>
parents:
76
diff
changeset
|
378 | self.current_operation.time=dreamtime.MonotonicTime.now() |
64 | 379 | self.state=State.ACTIVE |
380 | # Reset error status when starting a new operation | |
381 | self.errors=Errors.OK | |
382 | self.__update_status() | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
383 | |
7 | 384 | t_log.start() |
385 | t_res.start() | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
386 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
387 | |
49 | 388 | def __launch(self, op): |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
389 | self.logger.debug("Launching '%s'" % str(op.operation)) |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
390 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
391 | params=(config.borg_parameters |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
392 | +self.repository.borg_parameters |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
393 | +self.borg_parameters) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
394 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
395 | if op.operation==Operation.CREATE: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
396 | archive="%s::%s%s" % (self.repository.location, |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
397 | self.archive_prefix, |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
398 | self.archive_template) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
399 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
400 | self.__do_launch(op, archive, params.common, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
401 | params.create, self.paths) |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
402 | elif op.operation==Operation.PRUNE: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
403 | self.__do_launch(op, self.repository.location, params.common, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
404 | [{'prefix': self.archive_prefix}] + params.create) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
405 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
406 | elif op.operation==Operation.LIST: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
407 | self.__do_launch(op, self.repository.location, params.common, |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
408 | [{'prefix': self.archive_prefix}]) |
8 | 409 | else: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
410 | raise NotImplementedError("Invalid operation '%s'" % str(op.operation)) |
2 | 411 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
412 | # This must be called with self._cond held. |
64 | 413 | def __launch_and_wait(self): |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
414 | op=self.scheduled_operation |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
415 | if not op: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
416 | self.logger.debug("Queued operation aborted") |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
417 | else: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
418 | self.scheduled_operation=None |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
419 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
420 | self.__launch(op) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
421 | |
64 | 422 | self.__wait_finish() |
423 | ||
424 | def __wait_finish(self): | |
425 | # Wait for main logger thread to terminate, or for us to be terminated | |
426 | while not self.terminate and self.thread_res.is_alive(): | |
427 | self._cond.release() | |
428 | self.thread_res.join(JOIN_TIMEOUT) | |
429 | self._cond.acquire() | |
430 | ||
431 | # If terminate has been signalled, let outer termination handler | |
432 | # take care of things (Within this Backup class, it would be cleanest | |
433 | # to raise an exception instead, but in most other places it's better | |
434 | # to just check self._terminate, so we don't complicate things with | |
435 | # an extra exception.) | |
436 | if self._terminate: | |
437 | return | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
438 | |
64 | 439 | self.logger.debug('Waiting for borg and log subprocesses to terminate') |
440 | ||
441 | self._cond.release() | |
442 | self.thread_log.join() | |
443 | self._cond.acquire() | |
444 | ||
445 | if not self.borg_instance.wait(): | |
446 | self.logger.error('Borg subprocess did not terminate') | |
447 | self.errors=self.errors.combine(Errors.ERRORS) | |
448 | ||
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
449 | now=time.monotonic() |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
450 | self.last_operation_finished=now |
64 | 451 | self.thread_res=None |
452 | self.thread_log=None | |
453 | self.borg_instance=None | |
454 | self.current_operation=None | |
455 | self.state=State.INACTIVE | |
456 | self.__update_status() | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
457 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
458 | def __main_thread(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
459 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
460 | try: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
461 | while not self._terminate: |
64 | 462 | assert(not self.current_operation) |
463 | self.__main_thread_wait_schedule() | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
464 | if not self._terminate: |
64 | 465 | self.__main_thread_queue_and_launch() |
32 | 466 | except Exception as err: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
467 | self.logger.exception("Error with backup '%s'" % self.backup_name) |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
468 | self.errors=Errors.ERRORS |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
469 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
470 | self.state=State.INACTIVE |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
471 | self.scheduled_operation=None |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
472 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
473 | # Clean up to terminate: kill borg instance and communication threads |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
474 | if self.borg_instance: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
475 | self.logger.debug("Terminating a borg instance") |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
476 | self.borg_instance.terminate() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
477 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
478 | # Store threads to use outside lock |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
479 | thread_log=self.thread_log |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
480 | thread_res=self.thread_res |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
481 | self.thread_log=None |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
482 | self.thread_res=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
483 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
484 | self.logger.debug("Waiting for log and result threads to terminate") |
8 | 485 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
486 | if thread_log: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
487 | thread_log.join() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
488 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
489 | if thread_res: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
490 | thread_res.join() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
491 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
492 | # Main thread/2. Schedule next operation if there is no manually |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
493 | # requested one |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
494 | def __main_thread_wait_schedule(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
495 | op=None |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
496 | if not self.scheduled_operation: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
497 | op=self.__next_operation_unlocked() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
498 | if op: |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
499 | self.logger.info("Scheduling '%s' (detail: %s) in %d seconds [%s]" % |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
500 | (str(op.operation), op.detail or 'none', |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
501 | op.time.seconds_to(), |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
502 | op.time.__class__.__name__)) |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
503 | |
49 | 504 | self.scheduled_operation=op |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
505 | self.state=State.SCHEDULED |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
506 | self.__update_status() |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
507 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
508 | # Wait under scheduled wait |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
509 | self.scheduler.wait_until(op.time, self._cond, self.backup_name) |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
510 | else: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
511 | # Nothing scheduled - just wait |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
512 | self.logger.info("Waiting for manual scheduling") |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
513 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
514 | self.state=State.INACTIVE |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
515 | self.__update_status() |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
516 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
517 | self._cond.wait() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
518 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
519 | # Main thread/3. If there is a scheduled operation (it might have been |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
520 | # changed manually from 'op' created in __main_thread_wait_schedule above), |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
521 | # queue it on the repository, and launch the operation once repository |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
522 | # available |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
523 | def __main_thread_queue_and_launch(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
524 | if self.scheduled_operation: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
525 | self.logger.debug("Queuing") |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
526 | self.state=State.QUEUED |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
527 | self.__update_status() |
64 | 528 | res=self.repository.queue_action(self._cond, |
529 | action=self.__launch_and_wait, | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
530 | name=self.backup_name) |
64 | 531 | if not res and not self._terminate: |
532 | self.logger.debug("Queueing aborted") | |
533 | self.scheduled_operation=None | |
534 | self.state=State.INACTIVE | |
535 | self.__update_status() | |
8 | 536 | |
537 | def __next_operation_unlocked(self): | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
538 | if self.backup_interval==0: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
539 | # Manual backup |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
540 | return None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
541 | elif not self.last_create_finished: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
542 | # Don't know when last create finished: try to get it from |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
543 | # archive list. |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
544 | if not self.errors.ok() and self.retry_interval==0: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
545 | # Do not retry in case of errors if retry interval is zero |
10 | 546 | return None |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
547 | elif self.last_operation_finished: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
548 | # Attempt ater retry interval if some operation has been run |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
549 | # already |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
550 | tm=dreamtime.MonotonicTime(self.last_operation_finished+self.retry_interval) |
10 | 551 | else: |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
552 | # Nothing has been attempted: run immediately |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
553 | tm=dreamtime.MonotonicTime.now() |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
554 | return Operation(Operation.LIST, tm, reason='initial') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
555 | elif not self.errors.ok() or not self.last_create_finished: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
556 | # Retry create in case of errors |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
557 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
558 | if not self.errors.ok(): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
559 | reason='retry' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
560 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
561 | reason='initial' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
562 | |
10 | 563 | if self.retry_interval==0: |
564 | return None | |
565 | else: | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
566 | basetime = self.last_create_finished or time.monotonic() |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
567 | tm=dreamtime.MonotonicTime(basetime+self.retry_interval) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
568 | return Operation(Operation.CREATE, tm, reason=reason) |
5 | 569 | else: |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
570 | # All ok - run create at standard backup interval |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
571 | tm=self.timeclass.from_monotonic(self.last_create_when+self.backup_interval) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
572 | return Operation(Operation.CREATE, tm) |
10 | 573 | |
574 | def __status_unlocked(self): | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
575 | callback=self.__status_update_callback |
38 | 576 | |
10 | 577 | if self.current_operation: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
578 | status=Status(self, self.current_operation) |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
579 | elif self.scheduled_operation: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
580 | status=Status(self, self.scheduled_operation) |
10 | 581 | else: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
582 | status=Status(self) |
10 | 583 | |
584 | return status, callback | |
585 | ||
49 | 586 | def __update_status(self): |
587 | status, callback = self.__status_unlocked() | |
588 | if callback: | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
589 | #self._cond.release() |
49 | 590 | try: |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
591 | callback(status) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
592 | except Exception: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
593 | self.logger.exception("Status update error") |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
594 | #finally: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
595 | # self._cond.acquire() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
596 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
597 | # |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
598 | # Interface functions |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
599 | # |
49 | 600 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
601 | def set_status_update_callback(self, callback): |
49 | 602 | with self._cond: |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
603 | self.__status_update_callback=callback |
10 | 604 | |
49 | 605 | def status(self): |
606 | with self._cond: | |
607 | res=self.__status_unlocked() | |
608 | return res[0] | |
10 | 609 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
610 | def create(self): |
78
83b43987e61e
Renamed the "sleep" module "dreamtime"
Tuomo Valkonen <tuomov@iki.fi>
parents:
76
diff
changeset
|
611 | op=Operation(Operation.CREATE, dreamtime.MonotonicTime.now(), reason='manual') |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
612 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
613 | self.scheduled_operation=op |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
614 | self._cond.notify() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
615 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
616 | def prune(self): |
78
83b43987e61e
Renamed the "sleep" module "dreamtime"
Tuomo Valkonen <tuomov@iki.fi>
parents:
76
diff
changeset
|
617 | op=Operation(Operation.PRUNE, dreamtime.MonotonicTime.now(), reason='manual') |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
618 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
619 | self.scheduled_operation=op |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
620 | self._cond.notify() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
621 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
622 | # TODO: Decide exact (manual) abort mechanism. Perhaps two stages |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
623 | def abort(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
624 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
625 | if self.borg_instance: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
626 | self.borg_instance.terminate() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
627 |