instance.py

Sun, 28 Jan 2018 11:38:01 +0000

author
Tuomo Valkonen <tuomov@iki.fi>
date
Sun, 28 Jan 2018 11:38:01 +0000
changeset 79
b075b3db3044
parent 74
4f56142e7497
permissions
-rw-r--r--

Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.

0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
1 #
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
2 # Borgend borg launcher / processor
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
3 #
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
4
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
5 import json
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
6 import logging
20
fdfbe5d7b677 Keychain support and random fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 12
diff changeset
7 import os
79
b075b3db3044 Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents: 74
diff changeset
8 import loggers
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
9 from config import settings
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
10 from subprocess import Popen, PIPE
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
11
79
b075b3db3044 Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents: 74
diff changeset
12 logger=loggers.get(__name__)
31
b4b4bb7a2ec5 More logging detail
Tuomo Valkonen <tuomov@iki.fi>
parents: 29
diff changeset
13
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
14 necessary_opts=['--log-json', '--progress']
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
15
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
16 necessary_opts_for={
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
17 'create': ['--json'],
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
18 'info': ['--json'],
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
19 'list': ['--json'],
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
20 }
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
21
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
22 # Conversion of config into command line
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
23 def arglistify(args):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
24 flatten=lambda l: [item for sublist in l for item in sublist]
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
25 if args is None:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
26 return []
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
27 else:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
28 return flatten([['--' + key, str(d[key])] for d in args for key in d])
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
29
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
30 class BorgInstance:
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
31 def __init__(self, operation, archive_or_repository,
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
32 common_params, op_params, paths):
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
33 self.operation=operation;
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
34 self.archive_or_repository=archive_or_repository;
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
35 self.common_params=common_params
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
36 self.op_params=op_params
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
37 self.paths=paths
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
38
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
39 def construct_cmdline(self):
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
40 cmd=([settings['borg']['executable']]+necessary_opts+
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
41 arglistify(self.common_params)+
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
42 [self.operation])
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
43
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
44 if self.operation in necessary_opts_for:
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
45 cmd=cmd+necessary_opts_for[self.operation]
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
46
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
47 return (cmd+arglistify(self.op_params)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 34
diff changeset
48 +[self.archive_or_repository]+self.paths)
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
49
21
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 20
diff changeset
50 def launch(self, passphrase=None):
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
51 cmd=self.construct_cmdline()
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
52
31
b4b4bb7a2ec5 More logging detail
Tuomo Valkonen <tuomov@iki.fi>
parents: 29
diff changeset
53 logger.info('Launching ' + str(cmd))
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
54
32
06fc14211ba9 Error handling improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 31
diff changeset
55 # Set passphrase if not, or set to empty if not known, so borg
06fc14211ba9 Error handling improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 31
diff changeset
56 # won't hang waiting for it, which seems to happen even if we
06fc14211ba9 Error handling improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 31
diff changeset
57 # close stdin.
29
9dd2e5d57a7c Something weird going on
Tuomo Valkonen <tuomov@iki.fi>
parents: 24
diff changeset
58 env=os.environ.copy()
32
06fc14211ba9 Error handling improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 31
diff changeset
59 env['BORG_PASSPHRASE']=passphrase or ''
20
fdfbe5d7b677 Keychain support and random fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 12
diff changeset
60
24
94d58d514d69 Workaround to PYTHONPATH and PYTHONHOME being messed up by py2app.
Tuomo Valkonen <tuomov@iki.fi>
parents: 23
diff changeset
61 # Workaround: if launched is a standalone app created with py2app,
94d58d514d69 Workaround to PYTHONPATH and PYTHONHOME being messed up by py2app.
Tuomo Valkonen <tuomov@iki.fi>
parents: 23
diff changeset
62 # borg will fail unless Python environment is reset.
94d58d514d69 Workaround to PYTHONPATH and PYTHONHOME being messed up by py2app.
Tuomo Valkonen <tuomov@iki.fi>
parents: 23
diff changeset
63 # TODO: Of course, this will fail if the system needs the variables
94d58d514d69 Workaround to PYTHONPATH and PYTHONHOME being messed up by py2app.
Tuomo Valkonen <tuomov@iki.fi>
parents: 23
diff changeset
64 # PYTHONPATH or PYTHONHOME set to certain values.
29
9dd2e5d57a7c Something weird going on
Tuomo Valkonen <tuomov@iki.fi>
parents: 24
diff changeset
65 if '_PY2APP_LAUNCHED_' in env:
9dd2e5d57a7c Something weird going on
Tuomo Valkonen <tuomov@iki.fi>
parents: 24
diff changeset
66 val=env['_PY2APP_LAUNCHED_']
9dd2e5d57a7c Something weird going on
Tuomo Valkonen <tuomov@iki.fi>
parents: 24
diff changeset
67 if val=='1':
9dd2e5d57a7c Something weird going on
Tuomo Valkonen <tuomov@iki.fi>
parents: 24
diff changeset
68 del env['PYTHONPATH']
9dd2e5d57a7c Something weird going on
Tuomo Valkonen <tuomov@iki.fi>
parents: 24
diff changeset
69 del env['PYTHONHOME']
24
94d58d514d69 Workaround to PYTHONPATH and PYTHONHOME being messed up by py2app.
Tuomo Valkonen <tuomov@iki.fi>
parents: 23
diff changeset
70
20
fdfbe5d7b677 Keychain support and random fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 12
diff changeset
71 self.proc=Popen(cmd, env=env, stdout=PIPE, stderr=PIPE, stdin=PIPE)
12
16a8c63344c0 Fixed retry scheduling
Tuomo Valkonen <tuomov@iki.fi>
parents: 10
diff changeset
72
21
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 20
diff changeset
73 # We don't do passphrase input etc.
12
16a8c63344c0 Fixed retry scheduling
Tuomo Valkonen <tuomov@iki.fi>
parents: 10
diff changeset
74 self.proc.stdin.close()
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
75
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
76 def read_result(self):
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
77 stream=self.proc.stdout
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
78 line=stream.read(-1)
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
79 if line==b'':
31
b4b4bb7a2ec5 More logging detail
Tuomo Valkonen <tuomov@iki.fi>
parents: 29
diff changeset
80 logger.debug('Borg stdout pipe EOF?')
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
81 return None
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
82
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
83 try:
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
84 return json.loads(line)
32
06fc14211ba9 Error handling improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 31
diff changeset
85 except Exception as err:
31
b4b4bb7a2ec5 More logging detail
Tuomo Valkonen <tuomov@iki.fi>
parents: 29
diff changeset
86 logger.warning('JSON parse failed on: "%s"' % line)
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
87 return None
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
88
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
89 def read_log(self):
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
90 stream=self.proc.stderr
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
91 try:
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
92 line=stream.readline()
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
93 except err:
31
b4b4bb7a2ec5 More logging detail
Tuomo Valkonen <tuomov@iki.fi>
parents: 29
diff changeset
94 logger.debug('Pipe read failed: %s' % str(err))
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
95
21
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 20
diff changeset
96 return {'type': 'log_message',
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 20
diff changeset
97 'levelname': 'CRITICAL',
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 20
diff changeset
98 'name': 'borgend.instance.BorgInstance',
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 20
diff changeset
99 'msgid': 'Borgend.Exception',
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 20
diff changeset
100 'message': err}
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
101
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
102 if line==b'':
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
103
31
b4b4bb7a2ec5 More logging detail
Tuomo Valkonen <tuomov@iki.fi>
parents: 29
diff changeset
104 logger.debug('Borg stderr pipe EOF?')
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
105
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
106 return None
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
107
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
108 try:
6
46c89e5a219f subprocess improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
109 res=json.loads(line)
46c89e5a219f subprocess improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
110 if 'type' not in res:
46c89e5a219f subprocess improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
111 res['type']='UNKNOWN'
46c89e5a219f subprocess improvements
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
112 return res
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
113 except:
31
b4b4bb7a2ec5 More logging detail
Tuomo Valkonen <tuomov@iki.fi>
parents: 29
diff changeset
114 logger.debug('JSON parse failed on: "%s"' % str(line))
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
115
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
116 errmsg=line
7
e189d4a6cb8c Also listen to stdout
Tuomo Valkonen <tuomov@iki.fi>
parents: 6
diff changeset
117 for line in iter(stream.readline, b''):
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
118 errmsg=errmsg+line
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
119
21
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 20
diff changeset
120 return {'type': 'log_message',
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 20
diff changeset
121 'levelname': 'ERROR',
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 20
diff changeset
122 'name': 'borgend.instance.BorgInstance',
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 20
diff changeset
123 'msgid': 'Borgend.JSONFail',
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 20
diff changeset
124 'message': str(errmsg)}
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
125
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
126 def terminate(self):
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
127 self.proc.terminate()
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
128
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
129 def wait(self):
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
130 return self.proc.wait() is not None
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
131
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
132 def has_terminated(self):
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
133 return self.proc.poll() is not None
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
134

mercurial