backup.py

Fri, 19 Jan 2018 15:41:45 +0000

author
Tuomo Valkonen <tuomov@iki.fi>
date
Fri, 19 Jan 2018 15:41:45 +0000
changeset 4
d72c4844e791
parent 3
4cad934aa9ce
child 5
4c5514b2fa76
permissions
-rw-r--r--

Better borg output processing and some logging

1
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
1 #
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
2 # Borgend Backup instance
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
3 #
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
4
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
5 import config
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
6 import logging
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
7 from instance import BorgInstance
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
8 from queue import Queue
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
9 from threading import Thread, Lock
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
10
1
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
11 class Backup:
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
12
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
13 def __decode_config(self, cfg):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
14 loc0='backup target %d' % self.identifier
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
15
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
16 self.name=config.check_string(cfg, 'name', 'Name', loc0)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
17
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
18 self.loc='backup target "%s"' % self.name
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
19
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
20 self.repository=config.check_string(cfg, 'repository',
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
21 'Target repository', self.loc)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
22
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
23 self.archive_prefix=config.check_string(cfg, 'archive_prefix',
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
24 'Archive prefix', self.loc)
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
25
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
26 self.archive_template=config.check_string(cfg, 'archive_template',
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
27 'Archive template', self.loc)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
28
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
29 self.backup_interval=config.check_nonneg_int(cfg, 'backup_interval',
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
30 'Backup interval', self.loc,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
31 config.defaults['backup_interval'])
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
32
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
33 self.retry_interval=config.check_nonneg_int(cfg, 'retry_interval',
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
34 'Retry interval', self.loc,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
35 config.defaults['retry_interval'])
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
36
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
37 self.paths=config.check_nonempty_list_of_strings(cfg, 'paths', 'Paths', self.loc)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
38
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
39 self.create_parameters=config.check_list_of_dicts(cfg, 'create_parameters',
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
40 'Borg parameters', self.loc,
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
41 default=[])
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
42
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
43 self.prune_parameters=config.check_list_of_dicts(cfg, 'prune_parameters',
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
44 'Borg parameters', self.loc,
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
45 default=[])
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
46
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
47
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
48 def __init__(self, identifier, cfg):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
49 self.identifier=identifier
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
50
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
51 self.__decode_config(cfg)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
52
1
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
53 self.config=config
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
54 self.lastrun=None
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
55 self.borg_instance=None
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
56 self.thread=None
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
57 self.lock=Lock()
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
58
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
59 def __block_when_running(self):
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
60 self.lock.acquire()
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
61 not_running=self.borg_instance is None and self.thread is None
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
62 self.lock.release()
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
63 assert(not_running)
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
64
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
65 def __listener(self):
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
66 for status in iter(self.borg_instance.read, None):
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
67 t=status['type']
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
68 if t=='progress_percent':
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
69 pass
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
70 elif t=='archive_progress':
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
71 pass
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
72 elif t=='progress_message':
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
73 pass
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
74 elif t=='progress_percent':
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
75 pass
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
76 elif t=='file_status':
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
77 pass
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
78 elif t=='log_message':
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
79 pass
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
80 elif t=='exception':
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
81 pass
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
82 elif t=='unparsed_error':
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
83 pass
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
84 # What to do?
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
85 print(status)
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
86
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
87 logging.info('Borg subprocess finished; terminating listener thread')
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
88
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
89 self.lock.acquire()
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
90 self.borg_instance=None
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
91 self.thread=None
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
92 self.lock.release()
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
93
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
94 def __launch(self, queue, operation, archive_or_repository, *args):
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
95
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
96 inst=BorgInstance(operation, archive_or_repository, *args)
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
97 inst.launch()
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
98
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
99 t=Thread(target=self.__listener)
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
100 t.daemon=True
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
101
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
102 self.thread=t
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
103 self.borg_instance=inst
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
104 self.queue=queue
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
105
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
106 t.start()
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
107
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
108 def create(self, queue):
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
109 self.__block_when_running()
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
110
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
111 archive="%s::%s%s" % (self.repository,
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
112 self.archive_prefix,
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
113 self.archive_template)
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
114
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
115 self.__launch(queue, 'create', archive,
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
116 self.create_parameters, self.paths)
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
117
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
118 def prune(self, queue):
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
119 self.__block_when_running()
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
120 self.__launch(queue, 'prune', self.repository,
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
121 [{'prefix': self.archive_prefix}] + self.prune_parameters)
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
122
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
123 # TODO: Decide exact (manual) abort mechanism. Perhaps two stages
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
124 def abort(self):
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
125 self.lock.acquire()
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
126 if self.borg_instance:
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
127 self.borg_instance.terminate()
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
128 if self.thread:
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
129 self.thread.terminate()
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
130 self.lock.release()
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
131
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
132 def join(self):
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
133 if self.thread:
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
134 self.thread.join()
1
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
135
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
136
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
137

mercurial