Fri, 19 Jan 2018 15:41:45 +0000
Better borg output processing and some logging
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
1 | # |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
2 | # Borgend Backup instance |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
3 | # |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
4 | |
2 | 5 | import config |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
6 | import logging |
2 | 7 | from instance import BorgInstance |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
8 | from queue import Queue |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
9 | from threading import Thread, Lock |
2 | 10 | |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
11 | class Backup: |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
12 | |
2 | 13 | def __decode_config(self, cfg): |
14 | loc0='backup target %d' % self.identifier | |
15 | ||
16 | self.name=config.check_string(cfg, 'name', 'Name', loc0) | |
17 | ||
18 | self.loc='backup target "%s"' % self.name | |
19 | ||
20 | self.repository=config.check_string(cfg, 'repository', | |
21 | 'Target repository', self.loc) | |
22 | ||
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
23 | self.archive_prefix=config.check_string(cfg, 'archive_prefix', |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
24 | 'Archive prefix', self.loc) |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
25 | |
2 | 26 | self.archive_template=config.check_string(cfg, 'archive_template', |
27 | 'Archive template', self.loc) | |
28 | ||
29 | self.backup_interval=config.check_nonneg_int(cfg, 'backup_interval', | |
30 | 'Backup interval', self.loc, | |
31 | config.defaults['backup_interval']) | |
32 | ||
33 | self.retry_interval=config.check_nonneg_int(cfg, 'retry_interval', | |
34 | 'Retry interval', self.loc, | |
35 | config.defaults['retry_interval']) | |
36 | ||
37 | self.paths=config.check_nonempty_list_of_strings(cfg, 'paths', 'Paths', self.loc) | |
38 | ||
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
39 | self.create_parameters=config.check_list_of_dicts(cfg, 'create_parameters', |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
40 | 'Borg parameters', self.loc, |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
41 | default=[]) |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
42 | |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
43 | self.prune_parameters=config.check_list_of_dicts(cfg, 'prune_parameters', |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
44 | 'Borg parameters', self.loc, |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
45 | default=[]) |
2 | 46 | |
47 | ||
48 | def __init__(self, identifier, cfg): | |
49 | self.identifier=identifier | |
50 | ||
51 | self.__decode_config(cfg) | |
52 | ||
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
53 | self.config=config |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
54 | self.lastrun=None |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
55 | self.borg_instance=None |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
56 | self.thread=None |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
57 | self.lock=Lock() |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
58 | |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
59 | def __block_when_running(self): |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
60 | self.lock.acquire() |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
61 | not_running=self.borg_instance is None and self.thread is None |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
62 | self.lock.release() |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
63 | assert(not_running) |
2 | 64 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
65 | def __listener(self): |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
66 | for status in iter(self.borg_instance.read, None): |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
67 | t=status['type'] |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
68 | if t=='progress_percent': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
69 | pass |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
70 | elif t=='archive_progress': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
71 | pass |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
72 | elif t=='progress_message': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
73 | pass |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
74 | elif t=='progress_percent': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
75 | pass |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
76 | elif t=='file_status': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
77 | pass |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
78 | elif t=='log_message': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
79 | pass |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
80 | elif t=='exception': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
81 | pass |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
82 | elif t=='unparsed_error': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
83 | pass |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
84 | # What to do? |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
85 | print(status) |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
86 | |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
87 | logging.info('Borg subprocess finished; terminating listener thread') |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
88 | |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
89 | self.lock.acquire() |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
90 | self.borg_instance=None |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
91 | self.thread=None |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
92 | self.lock.release() |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
93 | |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
94 | def __launch(self, queue, operation, archive_or_repository, *args): |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
95 | |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
96 | inst=BorgInstance(operation, archive_or_repository, *args) |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
97 | inst.launch() |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
98 | |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
99 | t=Thread(target=self.__listener) |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
100 | t.daemon=True |
2 | 101 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
102 | self.thread=t |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
103 | self.borg_instance=inst |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
104 | self.queue=queue |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
105 | |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
106 | t.start() |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
107 | |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
108 | def create(self, queue): |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
109 | self.__block_when_running() |
2 | 110 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
111 | archive="%s::%s%s" % (self.repository, |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
112 | self.archive_prefix, |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
113 | self.archive_template) |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
114 | |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
115 | self.__launch(queue, 'create', archive, |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
116 | self.create_parameters, self.paths) |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
117 | |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
118 | def prune(self, queue): |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
119 | self.__block_when_running() |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
120 | self.__launch(queue, 'prune', self.repository, |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
121 | [{'prefix': self.archive_prefix}] + self.prune_parameters) |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
122 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
123 | # TODO: Decide exact (manual) abort mechanism. Perhaps two stages |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
124 | def abort(self): |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
125 | self.lock.acquire() |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
126 | if self.borg_instance: |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
127 | self.borg_instance.terminate() |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
128 | if self.thread: |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
129 | self.thread.terminate() |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
130 | self.lock.release() |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
131 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
132 | def join(self): |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
133 | if self.thread: |
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
134 | self.thread.join() |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
135 | |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
136 | |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
137 |