# HG changeset patch # User Tuomo Valkonen # Date 1516358461 0 # Node ID e343594c00148f894485bb4460b4e0097babe8f1 # Parent 4cdc9c1f6b281d224d800724689ae17e0911c5d2 basic config processing diff -r 4cdc9c1f6b28 -r e343594c0014 backup.py --- a/backup.py Thu Jan 18 23:01:16 2018 +0000 +++ b/backup.py Fri Jan 19 10:41:01 2018 +0000 @@ -2,12 +2,58 @@ # Borgend Backup instance # +import config +from instance import BorgInstance + class Backup: - def __init__(self, identifier, config): + def __decode_config(self, cfg): + loc0='backup target %d' % self.identifier + + self.name=config.check_string(cfg, 'name', 'Name', loc0) + + self.loc='backup target "%s"' % self.name + + self.repository=config.check_string(cfg, 'repository', + 'Target repository', self.loc) + + self.archive_template=config.check_string(cfg, 'archive_template', + 'Archive template', self.loc) + + self.backup_interval=config.check_nonneg_int(cfg, 'backup_interval', + 'Backup interval', self.loc, + config.defaults['backup_interval']) + + self.retry_interval=config.check_nonneg_int(cfg, 'retry_interval', + 'Retry interval', self.loc, + config.defaults['retry_interval']) + + self.paths=config.check_nonempty_list_of_strings(cfg, 'paths', 'Paths', self.loc) + + self.borg_parameters=config.check_list_of_dicts(cfg, 'borg_parameters', + 'Borg parameters', self.loc, + default=[]) + + + def __init__(self, identifier, cfg): + self.identifier=identifier + + self.__decode_config(cfg) + + self.config=config - self.identifier=identifier self.lastrun=None + self.current_instance=None + + def create(self): + if self.current_instance is not None: + raise AssertionError('%s running: cannot launch' % self.loc) + + archive="%s::%s" % (self.repository, self.archive_template) + + inst=BorgInstance(self.identifier, 'create', self.borg_parameters, + archive, self.paths) + print(inst.construct_cmdline()) diff -r 4cdc9c1f6b28 -r e343594c0014 borgend.py --- a/borgend.py Thu Jan 18 23:01:16 2018 +0000 +++ b/borgend.py Fri Jan 19 10:41:01 2018 +0000 @@ -4,11 +4,12 @@ from config import settings import ui import queue -import scheduler +#import scheduler if __name__ == "__main__": - print(settings) + #print(settings) #BorgendTray("Borgend").run() + pass backupconfigs=settings['backups'] backups=[None]*len(backupconfigs); @@ -16,5 +17,4 @@ for i in range(len(backupconfigs)): backups[i]=Backup(i, backupconfigs[i]) - -schedul \ No newline at end of file +backups[0].create() diff -r 4cdc9c1f6b28 -r e343594c0014 config.py --- a/config.py Thu Jan 18 23:01:16 2018 +0000 +++ b/config.py Fri Jan 19 10:41:01 2018 +0000 @@ -7,7 +7,95 @@ import os import xdg import string +from functools import reduce +# +# Defaults +# + +defaults={ + # Default: backup every 6 hours (21600 seconds) + 'backup_interval': 21600, + # Default: retry every 15 minutes if unable to connect / unfinished backup + 'retry_interval': 900, + # borg + 'borg': { + 'executable': 'borg', + 'common_parameters': [], + 'create_parameters': [], + } +} + + +# +# Type checking etc. +# + +def error(x): + raise AssertionError(x) + +def check_string(cfg, field, descr, loc, default=None): + return check_field(cfg, field, descr, loc, default, + lambda x: isinstance(x, str)) + +def check_dict(cfg, field, descr, loc, default=None): + return check_field(cfg, field, descr, loc, default, + lambda x: isinstance(x, dict)) + +def check_list(cfg, field, descr, loc, default=None): + return check_field(cfg, field, descr, loc, default, + lambda x: isinstance(x, list)) + +def is_list_of(x, chk): + if x is None: + return True + elif isinstance(x, list): + return reduce(lambda y, z: y and chk(z), x, True) + else: + return False + +def check_list_of_dicts(cfg, field, descr, loc, default=None): + return check_field(cfg, field, descr, loc, default, + lambda x: is_list_of(x, lambda z: isinstance(z, dict))) + +def check_list_of_strings(cfg, field, descr, loc, default=None): + return check_field(cfg, field, descr, loc, default, + lambda x: is_list_of(x, lambda z: isinstance(z, str))) + +def check_nonempty_list_of_strings(cfg, field, descr, loc): + return check_list_of_strings(cfg, field, descr, loc) and cfg[field] + + +def check_nonneg_int(cfg, field, descr, loc, default=None): + return check_field(cfg, field, descr, loc, default, + lambda x: isinstance(x, int) and x>=0) + +def check_field(cfg, field, descr, loc, default, check): + if field in cfg: + tmp=cfg[field] + if not check(tmp): + error("%s is of invalid type for %s" % (field, loc)) + return tmp + else: + if default is not None: + return default + else: + error("%s is not configured for %s" % (field, loc)) + +# +# Conversion of config into command line +# + +def arglistify(args): + flatten=lambda l: [item for sublist in l for item in sublist] + if args is None: + return [] + else: + return flatten([['--' + key, str(d[key])] for d in args for key in d]) + +# +# Load config on module load +# def expand_env(cfg, env): if isinstance(cfg, dict): @@ -17,8 +105,8 @@ elif isinstance(cfg, str): out=string.Template(cfg).substitute(os.environ) else: - out=cfg - + out=cfg + return out cfgfile=os.path.join(xdg.XDG_CONFIG_HOME, "borgend", "config.yaml") @@ -29,3 +117,23 @@ with io.open(cfgfile, 'r') as file: settings=expand_env(yaml.load(file), os.environ); +# +# Verify basic settings +# + +if 'borg' not in settings: + settings['borg']=defaults['borg'] +else: + def check_and_set(cfg, field, loc, defa, fn): + cfg[field]=fn(cfg, field, field, loc, defa[field]) + return cfg + + settings['borg']=check_and_set(settings['borg'], 'executable', 'borg', + defaults['borg'], check_string) + + settings['borg']=check_and_set(settings['borg'], 'common_parameters', 'borg', + defaults['borg'], check_list_of_dicts) + + settings['borg']=check_and_set(settings['borg'], 'create_parameters', 'borg', + defaults['borg'], check_list_of_dicts) + diff -r 4cdc9c1f6b28 -r e343594c0014 instance.py --- a/instance.py Thu Jan 18 23:01:16 2018 +0000 +++ b/instance.py Fri Jan 19 10:41:01 2018 +0000 @@ -3,12 +3,12 @@ # import json -import subprocess -import config -import Queue -import Thread +from subprocess import Popen, PIPE +from config import settings, arglistify +from queue import Queue +from threading import Thread -def linereader(stream, instance, queue) +def linereader(stream, instance, queue): # What to do on error? for line in iter(stream.readline, b''): status=json.loads(line) @@ -19,19 +19,25 @@ class BorgInstance: - def __init__(self, identifier, operation, args): + def __init__(self, identifier, operation, args, archive, argsl): self.identifier=identifier; self.operation=operation; self.args=args; + self.archive=archive; + self.argsl=argsl; def construct_cmdline(self): - ??? + cmd=([settings['borg']['executable'], '--log-json']+ + arglistify(settings['borg']['common_parameters'])+ + [self.operation]) + tmp1=self.operation+'_parameters' + if tmp1 in settings['borg']: + cmd=cmd+arglistify(settings['borg'][tmp1]) + return cmd+arglistify(self.args)+[self.archive]+self.argsl def launch(self, queue): # What to do with stderr? Is it needed? - self.proc=subprocess.Popen(self.construct_cmdline(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + self.proc=Popen(self.construct_cmdline(), stdout=PIPE, stderr=PIPE) linereaderargs=(self.proc.stdout, self, queue) self.t=Thread(target=linereader, args=linereaderargs) t.daemon=True