Wed, 05 Dec 2018 08:27:35 -0500
screenshot, typofixes
23 | 1 | #!/usr/local/bin/python3 |
0 | 2 | # |
89
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
3 | # Borgend by Tuomo Valkonen, 2018 |
0 | 4 | # |
89
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
5 | # This file implements basic configuration processing |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
6 | # |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
7 | |
0 | 8 | |
9 | import yaml | |
10 | import io | |
11 | import os | |
12 | import string | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
13 | import logging |
43
8f3ac19f11b6
Use platform package to detect whether to:
Tuomo Valkonen <tuomov@iki.fi>
parents:
34
diff
changeset
|
14 | import platform |
2 | 15 | from functools import reduce |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
16 | |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
17 | from . import branding |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
18 | from . import locations |
0 | 19 | |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
20 | logger=logging.getLogger(__name__) |
31 | 21 | |
2 | 22 | # |
23 | # Defaults | |
24 | # | |
25 | ||
26 | defaults={ | |
21 | 27 | # borg |
28 | # Default: backup every 6 hours (21600 seconds) | |
29 | 'backup_interval': 21600, | |
30 | # Default: retry every 15 minutes if unable to connect / unfinished backup | |
31 | 'retry_interval': 900, | |
97 | 32 | # Default: do not prune (0) |
33 | 'prune_interval': 0, | |
21 | 34 | # Extract passphrases at startup or on demand? |
30
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
35 | 'extract_passphrases_at_startup': True, |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
36 | # Do not insert a quit menu entry (useful for installing on computers of |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
37 | # inexperienced users) |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
38 | 'no_quit_menu_entry': False, |
21 | 39 | # Borg settings |
40 | 'borg': { | |
41 | 'executable': 'borg', | |
42 | 'common_parameters': [], | |
43 | 'create_parameters': [], | |
44 | 'prune_parameters': [], | |
2 | 45 | } |
46 | } | |
47 | ||
33
91421eeb4426
Use rumps.application_support instead of xdg paths. Also separated branding into config.py
Tuomo Valkonen <tuomov@iki.fi>
parents:
31
diff
changeset
|
48 | # |
2 | 49 | # Type checking etc. |
50 | # | |
51 | ||
52 | def error(x): | |
53 | raise AssertionError(x) | |
54 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
55 | def check_field(cfg, field, descr, loc, default, check): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
56 | if field in cfg: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
57 | tmp=cfg[field] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
58 | if not check(tmp): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
59 | error("%s is of invalid type for %s" % (field, loc)) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
60 | return tmp |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
61 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
62 | if default is not None: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
63 | return default |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
64 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
65 | error("%s is not configured for %s" % (field, loc)) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
66 | |
30
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
67 | def check_bool(cfg, field, descr, loc, default=None): |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
68 | return check_field(cfg, field, descr, loc, default, |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
69 | lambda x: isinstance(x, bool)) |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
70 | |
2 | 71 | def check_string(cfg, field, descr, loc, default=None): |
72 | return check_field(cfg, field, descr, loc, default, | |
73 | lambda x: isinstance(x, str)) | |
74 | ||
75 | def check_dict(cfg, field, descr, loc, default=None): | |
76 | return check_field(cfg, field, descr, loc, default, | |
77 | lambda x: isinstance(x, dict)) | |
78 | ||
79 | def check_list(cfg, field, descr, loc, default=None): | |
80 | return check_field(cfg, field, descr, loc, default, | |
81 | lambda x: isinstance(x, list)) | |
82 | ||
83 | def is_list_of(x, chk): | |
84 | if x is None: | |
85 | return True | |
86 | elif isinstance(x, list): | |
87 | return reduce(lambda y, z: y and chk(z), x, True) | |
88 | else: | |
89 | return False | |
90 | ||
91 | def check_list_of_dicts(cfg, field, descr, loc, default=None): | |
92 | return check_field(cfg, field, descr, loc, default, | |
93 | lambda x: is_list_of(x, lambda z: isinstance(z, dict))) | |
94 | ||
95 | def check_list_of_strings(cfg, field, descr, loc, default=None): | |
96 | return check_field(cfg, field, descr, loc, default, | |
97 | lambda x: is_list_of(x, lambda z: isinstance(z, str))) | |
98 | ||
99 | def check_nonempty_list_of_strings(cfg, field, descr, loc): | |
100 | return check_list_of_strings(cfg, field, descr, loc) and cfg[field] | |
101 | ||
102 | ||
103 | def check_nonneg_int(cfg, field, descr, loc, default=None): | |
104 | return check_field(cfg, field, descr, loc, default, | |
105 | lambda x: isinstance(x, int) and x>=0) | |
106 | ||
107 | ||
108 | # | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
109 | # Borg command line parameter configuration helper routines and classes |
2 | 110 | # |
111 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
112 | class BorgParameters: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
113 | def __init__(self, common, create, prune): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
114 | self.common=common or [] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
115 | self.create=create or [] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
116 | self.prune=prune or [] |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
117 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
118 | def from_config(cfg, loc): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
119 | common=check_list_of_dicts(cfg, 'common_parameters', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
120 | 'Borg parameters', loc, default=[]) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
121 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
122 | create=check_list_of_dicts(cfg, 'create_parameters', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
123 | 'Create parameters', loc, default=[]) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
124 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
125 | prune=check_list_of_dicts(cfg, 'prune_parameters', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
126 | 'Prune parameters', loc, default=[]) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
127 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
128 | return BorgParameters(common, create, prune) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
129 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
130 | def __add__(self, other): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
131 | common=self.common+other.common |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
132 | create=self.create+other.create |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
133 | prune=self.prune+other.prune |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
134 | return BorgParameters(common, create, prune) |
2 | 135 | |
136 | # | |
137 | # Load config on module load | |
138 | # | |
0 | 139 | |
140 | def expand_env(cfg, env): | |
141 | if isinstance(cfg, dict): | |
142 | out={key: expand_env(val, env) for key, val in cfg.items()} | |
143 | elif isinstance(cfg, list): | |
144 | out=[expand_env(val, env) for val in cfg] | |
145 | elif isinstance(cfg, str): | |
146 | out=string.Template(cfg).substitute(os.environ) | |
147 | else: | |
2 | 148 | out=cfg |
149 | ||
0 | 150 | return out |
151 | ||
79
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
152 | if not (os.path.exists(locations.cfgfile) and os.path.isfile(locations.cfgfile)): |
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
153 | raise SystemExit("Configuration file required: %s" % locations.cfgfile) |
0 | 154 | |
79
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
155 | logger.info("Reading configuration %s missing" % locations.cfgfile) |
0 | 156 | |
79
b075b3db3044
Cleaned up module organisation to simplify borgend.py and not have to import it in other modules.
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
157 | with io.open(locations.cfgfile, 'r') as file: |
0 | 158 | settings=expand_env(yaml.load(file), os.environ); |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
0
diff
changeset
|
159 | |
2 | 160 | # |
161 | # Verify basic settings | |
162 | # | |
163 | ||
21 | 164 | def check_and_set(cfg, field, loc, defa, fn): |
165 | cfg[field]=fn(cfg, field, field, loc, defa[field]) | |
166 | ||
167 | check_and_set(settings, 'backup_interval', 'top-level', defaults, check_nonneg_int) | |
168 | check_and_set(settings, 'retry_interval', 'top-level', defaults, check_nonneg_int) | |
30
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
169 | check_and_set(settings, 'extract_passphrases_at_startup', 'top-level', defaults, check_nonneg_int) |
3dd525652dc8
Added no_quit_menu_entry option
Tuomo Valkonen <tuomov@iki.fi>
parents:
23
diff
changeset
|
170 | check_and_set(settings, 'no_quit_menu_entry', 'top-level', defaults, check_bool) |
21 | 171 | check_and_set(settings, 'borg', 'top-level', defaults, check_dict) |
172 | # Check parameters within 'borg' | |
173 | if True: | |
174 | check_and_set(settings['borg'], 'executable', 'borg', | |
175 | defaults['borg'], check_string) | |
2 | 176 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
43
diff
changeset
|
177 | borg_parameters=BorgParameters.from_config(settings['borg'], "top-level") |
2 | 178 |