Sun, 28 Jan 2018 11:54:46 +0000
Better package-like organisation
23 | 1 | #!/usr/local/bin/python3 |
0 | 2 | # |
3 | # Borgend configuration loader | |
4 | # | |
5 | ||
6 | import yaml | |
7 | import io | |
8 | import os | |
9 | import string | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
10 | import logging |
43
8f3ac19f11b6
Use platform package to detect whether to:
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
11 | import platform |
2 | 12 | from functools import reduce |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
13 | |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
14 | from . import loggers |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
15 | from . import branding |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
16 | from . import locations |
0 | 17 | |
79
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
18 | logger=loggers.get(__name__) |
31 | 19 | |
2 | 20 | # |
21 | # Defaults | |
22 | # | |
23 | ||
24 | defaults={ | |
21 | 25 | # borg |
26 | # Default: backup every 6 hours (21600 seconds) | |
27 | 'backup_interval': 21600, | |
28 | # Default: retry every 15 minutes if unable to connect / unfinished backup | |
29 | 'retry_interval': 900, | |
30 | # Extract passphrases at startup or on demand? | |
30
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
31 | 'extract_passphrases_at_startup': True, |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
32 | # Do not insert a quit menu entry (useful for installing on computers of |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
33 | # inexperienced users) |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
34 | 'no_quit_menu_entry': False, |
21 | 35 | # Borg settings |
36 | 'borg': { | |
37 | 'executable': 'borg', | |
38 | 'common_parameters': [], | |
39 | 'create_parameters': [], | |
40 | 'prune_parameters': [], | |
2 | 41 | } |
42 | } | |
43 | ||
33
91421eeb4426
Use rumps.application_support instead of xdg paths. Also separated branding into config.py
Tuomo Valkonen <tuomov@iki.fi>
parents:
31
diff
changeset
|
44 | # |
2 | 45 | # Type checking etc. |
46 | # | |
47 | ||
48 | def error(x): | |
49 | raise AssertionError(x) | |
50 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
51 | def check_field(cfg, field, descr, loc, default, check): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
52 | if field in cfg: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
53 | tmp=cfg[field] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
54 | if not check(tmp): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
55 | error("%s is of invalid type for %s" % (field, loc)) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
56 | return tmp |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
57 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
58 | if default is not None: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
59 | return default |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
60 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
61 | error("%s is not configured for %s" % (field, loc)) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
62 | |
30
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
63 | def check_bool(cfg, field, descr, loc, default=None): |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
64 | return check_field(cfg, field, descr, loc, default, |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
65 | lambda x: isinstance(x, bool)) |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
66 | |
2 | 67 | def check_string(cfg, field, descr, loc, default=None): |
68 | return check_field(cfg, field, descr, loc, default, | |
69 | lambda x: isinstance(x, str)) | |
70 | ||
71 | def check_dict(cfg, field, descr, loc, default=None): | |
72 | return check_field(cfg, field, descr, loc, default, | |
73 | lambda x: isinstance(x, dict)) | |
74 | ||
75 | def check_list(cfg, field, descr, loc, default=None): | |
76 | return check_field(cfg, field, descr, loc, default, | |
77 | lambda x: isinstance(x, list)) | |
78 | ||
79 | def is_list_of(x, chk): | |
80 | if x is None: | |
81 | return True | |
82 | elif isinstance(x, list): | |
83 | return reduce(lambda y, z: y and chk(z), x, True) | |
84 | else: | |
85 | return False | |
86 | ||
87 | def check_list_of_dicts(cfg, field, descr, loc, default=None): | |
88 | return check_field(cfg, field, descr, loc, default, | |
89 | lambda x: is_list_of(x, lambda z: isinstance(z, dict))) | |
90 | ||
91 | def check_list_of_strings(cfg, field, descr, loc, default=None): | |
92 | return check_field(cfg, field, descr, loc, default, | |
93 | lambda x: is_list_of(x, lambda z: isinstance(z, str))) | |
94 | ||
95 | def check_nonempty_list_of_strings(cfg, field, descr, loc): | |
96 | return check_list_of_strings(cfg, field, descr, loc) and cfg[field] | |
97 | ||
98 | ||
99 | def check_nonneg_int(cfg, field, descr, loc, default=None): | |
100 | return check_field(cfg, field, descr, loc, default, | |
101 | lambda x: isinstance(x, int) and x>=0) | |
102 | ||
103 | ||
104 | # | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
105 | # Borg command line parameter configuration helper routines and classes |
2 | 106 | # |
107 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
108 | class BorgParameters: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
109 | def __init__(self, common, create, prune): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
110 | self.common=common or [] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
111 | self.create=create or [] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
112 | self.prune=prune or [] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
113 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
114 | def from_config(cfg, loc): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
115 | common=check_list_of_dicts(cfg, 'common_parameters', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
116 | 'Borg parameters', loc, default=[]) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
117 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
118 | create=check_list_of_dicts(cfg, 'create_parameters', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
119 | 'Create parameters', loc, default=[]) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
120 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
121 | prune=check_list_of_dicts(cfg, 'prune_parameters', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
122 | 'Prune parameters', loc, default=[]) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
123 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
124 | return BorgParameters(common, create, prune) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
125 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
126 | def __add__(self, other): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
127 | common=self.common+other.common |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
128 | create=self.create+other.create |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
129 | prune=self.prune+other.prune |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
130 | return BorgParameters(common, create, prune) |
2 | 131 | |
132 | # | |
133 | # Load config on module load | |
134 | # | |
0 | 135 | |
136 | def expand_env(cfg, env): | |
137 | if isinstance(cfg, dict): | |
138 | out={key: expand_env(val, env) for key, val in cfg.items()} | |
139 | elif isinstance(cfg, list): | |
140 | out=[expand_env(val, env) for val in cfg] | |
141 | elif isinstance(cfg, str): | |
142 | out=string.Template(cfg).substitute(os.environ) | |
143 | else: | |
2 | 144 | out=cfg |
145 | ||
0 | 146 | return out |
147 | ||
79
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
148 | if not (os.path.exists(locations.cfgfile) and os.path.isfile(locations.cfgfile)): |
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
149 | raise SystemExit("Configuration file required: %s" % locations.cfgfile) |
0 | 150 | |
79
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
151 | logger.info("Reading configuration %s missing" % locations.cfgfile) |
0 | 152 | |
79
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
153 | with io.open(locations.cfgfile, 'r') as file: |
0 | 154 | settings=expand_env(yaml.load(file), os.environ); |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
0
diff
changeset
|
155 | |
2 | 156 | # |
157 | # Verify basic settings | |
158 | # | |
159 | ||
21 | 160 | def check_and_set(cfg, field, loc, defa, fn): |
161 | cfg[field]=fn(cfg, field, field, loc, defa[field]) | |
162 | ||
163 | check_and_set(settings, 'backup_interval', 'top-level', defaults, check_nonneg_int) | |
164 | check_and_set(settings, 'retry_interval', 'top-level', defaults, check_nonneg_int) | |
30
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
165 | check_and_set(settings, 'extract_passphrases_at_startup', 'top-level', defaults, check_nonneg_int) |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
166 | check_and_set(settings, 'no_quit_menu_entry', 'top-level', defaults, check_bool) |
21 | 167 | check_and_set(settings, 'borg', 'top-level', defaults, check_dict) |
168 | # Check parameters within 'borg' | |
169 | if True: | |
170 | check_and_set(settings['borg'], 'executable', 'borg', | |
171 | defaults['borg'], check_string) | |
2 | 172 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
173 | borg_parameters=BorgParameters.from_config(settings['borg'], "top-level") |
2 | 174 |