config.py

Sun, 21 Jan 2018 02:44:00 +0000

author
Tuomo Valkonen <tuomov@iki.fi>
date
Sun, 21 Jan 2018 02:44:00 +0000
changeset 25
efa4c61c6f01
parent 23
86b42610d5a8
child 30
3dd525652dc8
permissions
-rw-r--r--

Sample configuration file added along with a few words of documentation

23
86b42610d5a8 #!python3 in script file
Tuomo Valkonen <tuomov@iki.fi>
parents: 22
diff changeset
1 #!/usr/local/bin/python3
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
2 #
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
3 # Borgend configuration loader
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
4 #
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
5
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
6 import yaml
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
7 import io
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
8 import os
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
9 import xdg
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
10 import string
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
11 import logging
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
12 from functools import reduce
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
13
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
14 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
15 # Defaults
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
16 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
17
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
18 defaults={
21
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
19 # borg
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
20 # Default: backup every 6 hours (21600 seconds)
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
21 'backup_interval': 21600,
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
22 # Default: retry every 15 minutes if unable to connect / unfinished backup
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
23 'retry_interval': 900,
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
24 # Extract passphrases at startup or on demand?
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
25 '__extract_passphrases_at_startup': True,
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
26 # Borg settings
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
27 'borg': {
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
28 'executable': 'borg',
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
29 'common_parameters': [],
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
30 'create_parameters': [],
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
31 'prune_parameters': [],
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
32 }
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
33 }
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
34
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
35
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
36 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
37 # Type checking etc.
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
38 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
39
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
40 def error(x):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
41 raise AssertionError(x)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
42
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
43 def check_string(cfg, field, descr, loc, default=None):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
44 return check_field(cfg, field, descr, loc, default,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
45 lambda x: isinstance(x, str))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
46
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
47 def check_dict(cfg, field, descr, loc, default=None):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
48 return check_field(cfg, field, descr, loc, default,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
49 lambda x: isinstance(x, dict))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
50
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
51 def check_list(cfg, field, descr, loc, default=None):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
52 return check_field(cfg, field, descr, loc, default,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
53 lambda x: isinstance(x, list))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
54
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
55 def is_list_of(x, chk):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
56 if x is None:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
57 return True
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
58 elif isinstance(x, list):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
59 return reduce(lambda y, z: y and chk(z), x, True)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
60 else:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
61 return False
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
62
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
63 def check_list_of_dicts(cfg, field, descr, loc, default=None):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
64 return check_field(cfg, field, descr, loc, default,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
65 lambda x: is_list_of(x, lambda z: isinstance(z, dict)))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
66
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
67 def check_list_of_strings(cfg, field, descr, loc, default=None):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
68 return check_field(cfg, field, descr, loc, default,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
69 lambda x: is_list_of(x, lambda z: isinstance(z, str)))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
70
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
71 def check_nonempty_list_of_strings(cfg, field, descr, loc):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
72 return check_list_of_strings(cfg, field, descr, loc) and cfg[field]
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
73
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
74
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
75 def check_nonneg_int(cfg, field, descr, loc, default=None):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
76 return check_field(cfg, field, descr, loc, default,
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
77 lambda x: isinstance(x, int) and x>=0)
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
78
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
79 def check_field(cfg, field, descr, loc, default, check):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
80 if field in cfg:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
81 tmp=cfg[field]
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
82 if not check(tmp):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
83 error("%s is of invalid type for %s" % (field, loc))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
84 return tmp
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
85 else:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
86 if default is not None:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
87 return default
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
88 else:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
89 error("%s is not configured for %s" % (field, loc))
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
90
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
91 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
92 # Conversion of config into command line
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
93 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
94
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
95 def arglistify(args):
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
96 flatten=lambda l: [item for sublist in l for item in sublist]
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
97 if args is None:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
98 return []
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
99 else:
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
100 return flatten([['--' + key, str(d[key])] for d in args for key in d])
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
101
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
102 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
103 # Load config on module load
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
104 #
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
105
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
106 def expand_env(cfg, env):
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
107 if isinstance(cfg, dict):
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
108 out={key: expand_env(val, env) for key, val in cfg.items()}
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
109 elif isinstance(cfg, list):
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
110 out=[expand_env(val, env) for val in cfg]
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
111 elif isinstance(cfg, str):
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
112 out=string.Template(cfg).substitute(os.environ)
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
113 else:
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
114 out=cfg
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
115
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
116 return out
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
117
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
118 cfgfile=os.path.join(xdg.XDG_CONFIG_HOME, "borgend", "config.yaml")
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
119
4
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
120 logging.info("Reading configuration file %s" % cfgfile)
d72c4844e791 Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents: 3
diff changeset
121
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
122 if not (os.path.exists(cfgfile) and os.path.isfile(cfgfile)):
22
c3e95212e3f0 py2app standalone app generation
Tuomo Valkonen <tuomov@iki.fi>
parents: 21
diff changeset
123 raise SystemExit('Configuration file required: {cfgfile}')
0
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
124
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
125 with io.open(cfgfile, 'r') as file:
f5aecaad0bcf Some rough drafting
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
126 settings=expand_env(yaml.load(file), os.environ);
1
4cdc9c1f6b28 basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents: 0
diff changeset
127
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
128 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
129 # Verify basic settings
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
130 #
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
131
21
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
132 def check_and_set(cfg, field, loc, defa, fn):
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
133 cfg[field]=fn(cfg, field, field, loc, defa[field])
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
134
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
135 def check_parameters(cmd):
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
136 check_and_set(settings['borg'], cmd+'_parameters',
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
137 'borg', defaults['borg'],
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
138 check_list_of_dicts)
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
139
21
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
140 check_and_set(settings, 'backup_interval', 'top-level', defaults, check_nonneg_int)
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
141 check_and_set(settings, 'retry_interval', 'top-level', defaults, check_nonneg_int)
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
142 check_and_set(settings, '__extract_passphrases_at_startup', 'top-level', defaults, check_nonneg_int)
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
143 check_and_set(settings, 'borg', 'top-level', defaults, check_dict)
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
144 # Check parameters within 'borg'
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
145 if True:
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
146 check_and_set(settings['borg'], 'executable', 'borg',
c36e549a7f12 Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents: 4
diff changeset
147 defaults['borg'], check_string)
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
148
3
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
149 check_parameters('common')
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
150 check_parameters('create')
4cad934aa9ce Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents: 2
diff changeset
151 check_parameters('prune')
2
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
152
e343594c0014 basic config processing
Tuomo Valkonen <tuomov@iki.fi>
parents: 1
diff changeset
153

mercurial