Wed, 07 Feb 2018 20:39:01 +0000
Time snapshot fixes.
Python's default arguments are purely idiotic (aka. pythonic): generated
only once. This makes sense in a purely functional language, which Python
lightyears away from, but severely limits their usefulness in an imperative
language. Decorators also seem clumsy for this, as one would have to tell
the number of positional arguments for things to work nice, being able to
pass the snapshot both positionally and as keyword. No luck.
So have to do things the old-fashioned hard way.
23 | 1 | #!/usr/local/bin/python3 |
0 | 2 | # |
89
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
3 | # Borgend by Tuomo Valkonen, 2018 |
0 | 4 | # |
89
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
5 | # This file implements basic configuration processing |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
6 | # |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
7 | |
0 | 8 | |
9 | import yaml | |
10 | import io | |
11 | import os | |
12 | import string | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
13 | import logging |
43
8f3ac19f11b6
Use platform package to detect whether to:
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
14 | import platform |
2 | 15 | from functools import reduce |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
16 | |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
17 | from . import branding |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
18 | from . import locations |
0 | 19 | |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
20 | logger=logging.getLogger(__name__) |
31 | 21 | |
2 | 22 | # |
23 | # Defaults | |
24 | # | |
25 | ||
26 | defaults={ | |
21 | 27 | # borg |
28 | # Default: backup every 6 hours (21600 seconds) | |
29 | 'backup_interval': 21600, | |
30 | # Default: retry every 15 minutes if unable to connect / unfinished backup | |
31 | 'retry_interval': 900, | |
97 | 32 | # Default: do not prune (0) |
33 | 'prune_interval': 0, | |
21 | 34 | # Extract passphrases at startup or on demand? |
30
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
35 | 'extract_passphrases_at_startup': True, |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
36 | # Do not insert a quit menu entry (useful for installing on computers of |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
37 | # inexperienced users) |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
38 | 'no_quit_menu_entry': False, |
21 | 39 | # Borg settings |
40 | 'borg': { | |
41 | 'executable': 'borg', | |
42 | 'common_parameters': [], | |
43 | 'create_parameters': [], | |
44 | 'prune_parameters': [], | |
2 | 45 | } |
46 | } | |
47 | ||
33
91421eeb4426
Use rumps.application_support instead of xdg paths. Also separated branding into config.py
Tuomo Valkonen <tuomov@iki.fi>
parents:
31
diff
changeset
|
48 | # |
2 | 49 | # Type checking etc. |
50 | # | |
51 | ||
52 | def error(x): | |
53 | raise AssertionError(x) | |
54 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
55 | def check_field(cfg, field, descr, loc, default, check): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
56 | if field in cfg: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
57 | tmp=cfg[field] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
58 | if not check(tmp): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
59 | error("%s is of invalid type for %s" % (field, loc)) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
60 | return tmp |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
61 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
62 | if default is not None: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
63 | return default |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
64 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
65 | error("%s is not configured for %s" % (field, loc)) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
66 | |
30
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
67 | def check_bool(cfg, field, descr, loc, default=None): |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
68 | return check_field(cfg, field, descr, loc, default, |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
69 | lambda x: isinstance(x, bool)) |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
70 | |
2 | 71 | def check_string(cfg, field, descr, loc, default=None): |
72 | return check_field(cfg, field, descr, loc, default, | |
73 | lambda x: isinstance(x, str)) | |
74 | ||
75 | def check_dict(cfg, field, descr, loc, default=None): | |
76 | return check_field(cfg, field, descr, loc, default, | |
77 | lambda x: isinstance(x, dict)) | |
78 | ||
79 | def check_list(cfg, field, descr, loc, default=None): | |
80 | return check_field(cfg, field, descr, loc, default, | |
81 | lambda x: isinstance(x, list)) | |
82 | ||
83 | def is_list_of(x, chk): | |
84 | if x is None: | |
85 | return True | |
86 | elif isinstance(x, list): | |
87 | return reduce(lambda y, z: y and chk(z), x, True) | |
88 | else: | |
89 | return False | |
90 | ||
91 | def check_list_of_dicts(cfg, field, descr, loc, default=None): | |
92 | return check_field(cfg, field, descr, loc, default, | |
93 | lambda x: is_list_of(x, lambda z: isinstance(z, dict))) | |
94 | ||
95 | def check_list_of_strings(cfg, field, descr, loc, default=None): | |
96 | return check_field(cfg, field, descr, loc, default, | |
97 | lambda x: is_list_of(x, lambda z: isinstance(z, str))) | |
98 | ||
99 | def check_nonempty_list_of_strings(cfg, field, descr, loc): | |
100 | return check_list_of_strings(cfg, field, descr, loc) and cfg[field] | |
101 | ||
102 | ||
103 | def check_nonneg_int(cfg, field, descr, loc, default=None): | |
104 | return check_field(cfg, field, descr, loc, default, | |
105 | lambda x: isinstance(x, int) and x>=0) | |
106 | ||
107 | ||
108 | # | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
109 | # Borg command line parameter configuration helper routines and classes |
2 | 110 | # |
111 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
112 | class BorgParameters: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
113 | def __init__(self, common, create, prune): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
114 | self.common=common or [] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
115 | self.create=create or [] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
116 | self.prune=prune or [] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
117 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
118 | def from_config(cfg, loc): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
119 | common=check_list_of_dicts(cfg, 'common_parameters', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
120 | 'Borg parameters', loc, default=[]) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
121 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
122 | create=check_list_of_dicts(cfg, 'create_parameters', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
123 | 'Create parameters', loc, default=[]) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
124 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
125 | prune=check_list_of_dicts(cfg, 'prune_parameters', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
126 | 'Prune parameters', loc, default=[]) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
127 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
128 | return BorgParameters(common, create, prune) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
129 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
130 | def __add__(self, other): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
131 | common=self.common+other.common |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
132 | create=self.create+other.create |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
133 | prune=self.prune+other.prune |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
134 | return BorgParameters(common, create, prune) |
2 | 135 | |
136 | # | |
137 | # Load config on module load | |
138 | # | |
0 | 139 | |
140 | def expand_env(cfg, env): | |
141 | if isinstance(cfg, dict): | |
142 | out={key: expand_env(val, env) for key, val in cfg.items()} | |
143 | elif isinstance(cfg, list): | |
144 | out=[expand_env(val, env) for val in cfg] | |
145 | elif isinstance(cfg, str): | |
146 | out=string.Template(cfg).substitute(os.environ) | |
147 | else: | |
2 | 148 | out=cfg |
149 | ||
0 | 150 | return out |
151 | ||
79
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
152 | if not (os.path.exists(locations.cfgfile) and os.path.isfile(locations.cfgfile)): |
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
153 | raise SystemExit("Configuration file required: %s" % locations.cfgfile) |
0 | 154 | |
79
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
155 | logger.info("Reading configuration %s missing" % locations.cfgfile) |
0 | 156 | |
79
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
157 | with io.open(locations.cfgfile, 'r') as file: |
0 | 158 | settings=expand_env(yaml.load(file), os.environ); |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
0
diff
changeset
|
159 | |
2 | 160 | # |
161 | # Verify basic settings | |
162 | # | |
163 | ||
21 | 164 | def check_and_set(cfg, field, loc, defa, fn): |
165 | cfg[field]=fn(cfg, field, field, loc, defa[field]) | |
166 | ||
167 | check_and_set(settings, 'backup_interval', 'top-level', defaults, check_nonneg_int) | |
168 | check_and_set(settings, 'retry_interval', 'top-level', defaults, check_nonneg_int) | |
30
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
169 | check_and_set(settings, 'extract_passphrases_at_startup', 'top-level', defaults, check_nonneg_int) |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
170 | check_and_set(settings, 'no_quit_menu_entry', 'top-level', defaults, check_bool) |
21 | 171 | check_and_set(settings, 'borg', 'top-level', defaults, check_dict) |
172 | # Check parameters within 'borg' | |
173 | if True: | |
174 | check_and_set(settings['borg'], 'executable', 'borg', | |
175 | defaults['borg'], check_string) | |
2 | 176 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
177 | borg_parameters=BorgParameters.from_config(settings['borg'], "top-level") |
2 | 178 |