instance.py

Sat, 20 Jan 2018 14:04:51 +0000

author
Tuomo Valkonen <tuomov@iki.fi>
date
Sat, 20 Jan 2018 14:04:51 +0000
changeset 7
e189d4a6cb8c
parent 6
46c89e5a219f
child 10
76dbfb06eba0
permissions
-rw-r--r--

Also listen to stdout

0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
1 #
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
2 # Borgend borg launcher / processor
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
3 #
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
4
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
5 import json
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
6 import logging
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
7 from subprocess import Popen, PIPE
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
8 from config import settings, arglistify
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
9
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
10 necessary_opts=['--log-json', '--progress']
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
11
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
12 necessary_opts_for={
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
13 'create': ['--json'],
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
14 'info': ['--json'],
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
15 'list': ['--json'],
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
16 }
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
17
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
18 class BorgInstance:
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
19 def __init__(self, operation, archive_or_repository, args, argsl):
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
20 self.operation=operation;
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
21 self.args=args;
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
22 self.archive_or_repository=archive_or_repository;
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
23 self.argsl=argsl;
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
24
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
25 def construct_cmdline(self):
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
26 cmd=([settings['borg']['executable']]+necessary_opts+
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
27 arglistify(settings['borg']['common_parameters'])+
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
28 [self.operation])
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
29 tmp1=self.operation+'_parameters'
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
30 if tmp1 in settings['borg']:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
31 cmd=cmd+arglistify(settings['borg'][tmp1])
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
32
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
33 if self.operation in necessary_opts_for:
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
34 cmd=cmd+necessary_opts_for[self.operation]
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
35
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
36 return cmd+arglistify(self.args)+[self.archive_or_repository]+self.argsl
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
37
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
38 def launch(self):
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
39 cmd=self.construct_cmdline()
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
40
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
41 logging.info('Launching ' + str(cmd))
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
42
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
43 self.proc=Popen(cmd, stdout=PIPE, stderr=PIPE)
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
44
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
45 def read_result(self):
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
46 stream=self.proc.stdout
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
47 line=stream.read(-1)
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
48 if line==b'':
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
49 logging.debug('Borg stdout pipe EOF?')
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
50 return None
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
51
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
52 try:
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
53 return json.loads(line)
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
54 except:
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
55 logging.warning('JSON parse failed on: "%s"' % line)
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
56 return None
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
57
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
58 def read_log(self):
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
59 stream=self.proc.stderr
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
60 try:
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
61 line=stream.readline()
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
62 except err:
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
63 logging.debug('Pipe read failed: %s' % str(err))
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
64
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
65 return {'type': 'exception', 'exception': err}
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
66
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
67 if line==b'':
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
68
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
69 logging.debug('Borg stderr pipe EOF?')
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
70
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
71 return None
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
72
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
73 try:
6
46c89e5a219f subprocess improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
74 res=json.loads(line)
46c89e5a219f subprocess improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
75 if 'type' not in res:
46c89e5a219f subprocess improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
76 res['type']='UNKNOWN'
46c89e5a219f subprocess improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
77 return res
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
78 except:
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
79 logging.debug('JSON parse failed on: "%s"' % line)
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
80
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
81 errmsg=line
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
82 for line in iter(stream.readline, b''):
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
83 errmsg=errmsg+line
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
84
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
85 return {'type': 'unparsed_error', 'message': str(errmsg)}
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
86
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
87 def terminate(self):
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
88 self.proc.terminate()
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
89
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
90 def wait(self):
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
91 return self.proc.wait() is not None
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
92
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
93 def has_terminated(self):
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
94 return self.proc.poll() is not None
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
95

mercurial