Mon, 29 Jan 2018 14:17:03 +0000
Fixes to borg output processing after testing on a Linux system
0 | 1 | # |
89
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
2 | # Borgend by Tuomo Valkonen, 2018 |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
3 | # |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
4 | # This file implements a Borg launching interface. |
0 | 5 | # |
6 | ||
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
7 | import os |
0 | 8 | import json |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
9 | import logging |
2 | 10 | from subprocess import Popen, PIPE |
0 | 11 | |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
12 | from .config import settings |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
13 | |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
14 | logger=logging.getLogger(__name__) |
31 | 15 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
16 | necessary_opts=['--log-json', '--progress'] |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
17 | |
7 | 18 | necessary_opts_for={ |
19 | 'create': ['--json'], | |
20 | 'info': ['--json'], | |
21 | 'list': ['--json'], | |
22 | } | |
23 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
24 | # Conversion of config into command line |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
25 | def arglistify(args): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
26 | flatten=lambda l: [item for sublist in l for item in sublist] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
27 | if args is None: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
28 | return [] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
29 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
30 | return flatten([['--' + key, str(d[key])] for d in args for key in d]) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
31 | |
0 | 32 | class BorgInstance: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
33 | def __init__(self, operation, archive_or_repository, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
34 | common_params, op_params, paths): |
0 | 35 | self.operation=operation; |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
36 | self.archive_or_repository=archive_or_repository; |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
37 | self.common_params=common_params |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
38 | self.op_params=op_params |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
39 | self.paths=paths |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
40 | self.proc=None |
0 | 41 | |
42 | def construct_cmdline(self): | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
43 | cmd=([settings['borg']['executable']]+necessary_opts+ |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
44 | arglistify(self.common_params)+ |
2 | 45 | [self.operation]) |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
46 | |
7 | 47 | if self.operation in necessary_opts_for: |
48 | cmd=cmd+necessary_opts_for[self.operation] | |
49 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
50 | return (cmd+arglistify(self.op_params) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
51 | +[self.archive_or_repository]+self.paths) |
0 | 52 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
53 | def launch(self, passphrase=None): |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
54 | cmd=self.construct_cmdline() |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
55 | |
31 | 56 | logger.info('Launching ' + str(cmd)) |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
57 | |
32 | 58 | # Set passphrase if not, or set to empty if not known, so borg |
59 | # won't hang waiting for it, which seems to happen even if we | |
60 | # close stdin. | |
29 | 61 | env=os.environ.copy() |
32 | 62 | env['BORG_PASSPHRASE']=passphrase or '' |
20
fdfbe5d7b677
Keychain support and random fixes
Tuomo Valkonen <tuomov@iki.fi>
parents:
12
diff
changeset
|
63 | |
24
94d58d514d69
Workaround to PYTHONPATH and PYTHONHOME being messed up by py2app.
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
64 | # Workaround: if launched is a standalone app created with py2app, |
94d58d514d69
Workaround to PYTHONPATH and PYTHONHOME being messed up by py2app.
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
65 | # borg will fail unless Python environment is reset. |
94d58d514d69
Workaround to PYTHONPATH and PYTHONHOME being messed up by py2app.
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
66 | # TODO: Of course, this will fail if the system needs the variables |
94d58d514d69
Workaround to PYTHONPATH and PYTHONHOME being messed up by py2app.
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
67 | # PYTHONPATH or PYTHONHOME set to certain values. |
29 | 68 | if '_PY2APP_LAUNCHED_' in env: |
69 | val=env['_PY2APP_LAUNCHED_'] | |
70 | if val=='1': | |
71 | del env['PYTHONPATH'] | |
72 | del env['PYTHONHOME'] | |
24
94d58d514d69
Workaround to PYTHONPATH and PYTHONHOME being messed up by py2app.
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
73 | |
20
fdfbe5d7b677
Keychain support and random fixes
Tuomo Valkonen <tuomov@iki.fi>
parents:
12
diff
changeset
|
74 | self.proc=Popen(cmd, env=env, stdout=PIPE, stderr=PIPE, stdin=PIPE) |
12 | 75 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
76 | # We don't do passphrase input etc. |
12 | 77 | self.proc.stdin.close() |
0 | 78 | |
7 | 79 | def read_result(self): |
80 | stream=self.proc.stdout | |
95
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
81 | try: |
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
82 | line=stream.read(-1) |
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
83 | except Exception: |
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
84 | logger.exception('Borg stdout pipe read failed') |
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
85 | |
7 | 86 | if line==b'': |
95
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
87 | #logger.debug('Borg stdout pipe EOF?') |
7 | 88 | return None |
89 | ||
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
90 | try: |
95
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
91 | return json.loads(line.decode()) |
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
92 | except Exception: |
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
93 | logger.exception('JSON parse failed on: %s' % str(line)) |
7 | 94 | return None |
95 | ||
96 | def read_log(self): | |
97 | stream=self.proc.stderr | |
98 | try: | |
99 | line=stream.readline() | |
95
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
100 | except Exception: |
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
101 | logger.exception('Pipe stderr pipe read failed') |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
102 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
103 | return {'type': 'log_message', |
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
104 | 'levelname': 'CRITICAL', |
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
105 | 'name': 'borgend.instance.BorgInstance', |
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
106 | 'msgid': 'Borgend.Exception', |
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
107 | 'message': err} |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
108 | |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
109 | if line==b'': |
95
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
110 | #logger.debug('Borg stderr pipe EOF?') |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
111 | return None |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
112 | |
95
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
113 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
114 | try: |
95
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
115 | res=json.loads(line.decode()) |
6 | 116 | if 'type' not in res: |
117 | res['type']='UNKNOWN' | |
118 | return res | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
119 | except: |
95
41bd7025532f
Fixes to borg output processing after testing on a Linux system
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
120 | logger.exception('JSON parse failed on: %s' % str(line)) |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
121 | |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
122 | errmsg=line |
7 | 123 | for line in iter(stream.readline, b''): |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
124 | errmsg=errmsg+line |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
125 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
126 | return {'type': 'log_message', |
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
127 | 'levelname': 'ERROR', |
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
128 | 'name': 'borgend.instance.BorgInstance', |
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
129 | 'msgid': 'Borgend.JSONFail', |
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
130 | 'message': str(errmsg)} |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
131 | |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
132 | def terminate(self): |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
133 | if self.proc: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
134 | self.proc.terminate() |
0 | 135 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
136 | # Returns True if has terminated |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
137 | def wait(self, timeout=None): |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
138 | if self.proc: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
139 | return self.proc.wait(timeout=timeout) is not None |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
140 | else: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
141 | return True |
0 | 142 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
143 | def has_terminated(self): |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
144 | return not self.proc or (self.proc.poll() is not None) |
0 | 145 |