config.py

Sat, 20 Jan 2018 21:52:16 +0000

author
Tuomo Valkonen <tuomov@iki.fi>
date
Sat, 20 Jan 2018 21:52:16 +0000
changeset 15
aaf1a281a3fe
parent 4
d72c4844e791
child 21
c36e549a7f12
permissions
-rw-r--r--

More error reporting

0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
1 #
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
2 # Borgend configuration loader
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
3 #
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
4
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
5 import yaml
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
6 import io
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
7 import os
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
8 import xdg
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
9 import string
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
10 import logging
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
11 from functools import reduce
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
12
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
13 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
14 # Defaults
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
15 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
16
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
17 defaults={
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
18 # Default: backup every 6 hours (21600 seconds)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
19 'backup_interval': 21600,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
20 # Default: retry every 15 minutes if unable to connect / unfinished backup
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
21 'retry_interval': 900,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
22 # borg
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
23 'borg': {
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
24 'executable': 'borg',
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
25 'common_parameters': [],
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
26 'create_parameters': [],
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
27 'prune_parameters': [],
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
28 }
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
29 }
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
30
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
31
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
32 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
33 # Type checking etc.
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
34 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
35
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
36 def error(x):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
37 raise AssertionError(x)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
38
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
39 def check_string(cfg, field, descr, loc, default=None):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
40 return check_field(cfg, field, descr, loc, default,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
41 lambda x: isinstance(x, str))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
42
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
43 def check_dict(cfg, field, descr, loc, default=None):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
44 return check_field(cfg, field, descr, loc, default,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
45 lambda x: isinstance(x, dict))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
46
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
47 def check_list(cfg, field, descr, loc, default=None):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
48 return check_field(cfg, field, descr, loc, default,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
49 lambda x: isinstance(x, list))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
50
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
51 def is_list_of(x, chk):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
52 if x is None:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
53 return True
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
54 elif isinstance(x, list):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
55 return reduce(lambda y, z: y and chk(z), x, True)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
56 else:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
57 return False
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
58
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
59 def check_list_of_dicts(cfg, field, descr, loc, default=None):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
60 return check_field(cfg, field, descr, loc, default,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
61 lambda x: is_list_of(x, lambda z: isinstance(z, dict)))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
62
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
63 def check_list_of_strings(cfg, field, descr, loc, default=None):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
64 return check_field(cfg, field, descr, loc, default,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
65 lambda x: is_list_of(x, lambda z: isinstance(z, str)))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
66
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
67 def check_nonempty_list_of_strings(cfg, field, descr, loc):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
68 return check_list_of_strings(cfg, field, descr, loc) and cfg[field]
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
69
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
70
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
71 def check_nonneg_int(cfg, field, descr, loc, default=None):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
72 return check_field(cfg, field, descr, loc, default,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
73 lambda x: isinstance(x, int) and x>=0)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
74
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
75 def check_field(cfg, field, descr, loc, default, check):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
76 if field in cfg:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
77 tmp=cfg[field]
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
78 if not check(tmp):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
79 error("%s is of invalid type for %s" % (field, loc))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
80 return tmp
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
81 else:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
82 if default is not None:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
83 return default
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
84 else:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
85 error("%s is not configured for %s" % (field, loc))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
86
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
87 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
88 # Conversion of config into command line
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
89 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
90
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
91 def arglistify(args):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
92 flatten=lambda l: [item for sublist in l for item in sublist]
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
93 if args is None:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
94 return []
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
95 else:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
96 return flatten([['--' + key, str(d[key])] for d in args for key in d])
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
97
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
98 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
99 # Load config on module load
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
100 #
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
101
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
102 def expand_env(cfg, env):
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
103 if isinstance(cfg, dict):
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
104 out={key: expand_env(val, env) for key, val in cfg.items()}
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
105 elif isinstance(cfg, list):
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
106 out=[expand_env(val, env) for val in cfg]
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
107 elif isinstance(cfg, str):
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
108 out=string.Template(cfg).substitute(os.environ)
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
109 else:
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
110 out=cfg
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
111
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
112 return out
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
113
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
114 cfgfile=os.path.join(xdg.XDG_CONFIG_HOME, "borgend", "config.yaml")
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
115
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
116 logging.info("Reading configuration file %s" % cfgfile)
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
117
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
118 if not (os.path.exists(cfgfile) and os.path.isfile(cfgfile)):
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
119 raise SystemExit(f'Configuration file required: {cfgfile}')
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
120
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
121 with io.open(cfgfile, 'r') as file:
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
122 settings=expand_env(yaml.load(file), os.environ);
1
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents: 0
diff changeset
123
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
124 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
125 # Verify basic settings
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
126 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
127
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
128 if 'borg' not in settings:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
129 settings['borg']=defaults['borg']
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
130 else:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
131 def check_and_set(cfg, field, loc, defa, fn):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
132 cfg[field]=fn(cfg, field, field, loc, defa[field])
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
133 return cfg
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
134
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
135 def check_parameters(cmd):
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
136 settings['borg']=check_and_set(settings['borg'], cmd+'_parameters',
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
137 'borg', defaults['borg'],
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
138 check_list_of_dicts)
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
139
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
140 settings['borg']=check_and_set(settings['borg'], 'executable', 'borg',
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
141 defaults['borg'], check_string)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
142
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
143 check_parameters('common')
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
144 check_parameters('create')
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
145 check_parameters('prune')
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
146
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
147

mercurial