From 06a3a38efa80029be8351821697acb86a516efac Mon Sep 17 00:00:00 2001 From: maartenbreddels Date: Fri, 1 Sep 2017 15:15:08 +0200 Subject: [PATCH 1/4] allow default json files in a .d directory --- traitlets/config/manager.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/traitlets/config/manager.py b/traitlets/config/manager.py index 5e5ebde9..6a2ff1e0 100644 --- a/traitlets/config/manager.py +++ b/traitlets/config/manager.py @@ -9,7 +9,7 @@ from six import PY3 from traitlets.config import LoggingConfigurable -from traitlets.traitlets import Unicode +from traitlets.traitlets import Unicode, Bool def recursive_update(target, new): @@ -36,10 +36,12 @@ def recursive_update(target, new): class BaseJSONConfigManager(LoggingConfigurable): """General JSON config manager - Deals with persisting/storing config in a json file + Deals with persisting/storing config in a json file with optionally + default values in a {section_name}.d directory. """ config_dir = Unicode('.') + read_directory = Bool(True) def ensure_config_dir_exists(self): try: @@ -51,6 +53,9 @@ def ensure_config_dir_exists(self): def file_name(self, section_name): return os.path.join(self.config_dir, section_name+'.json') + def directory(self, section_name): + return os.path.join(self.config_dir, section_name+'.d') + def get(self, section_name): """Retrieve the config data for the specified section. @@ -58,11 +63,23 @@ def get(self, section_name): doesn't exist. """ filename = self.file_name(section_name) + data = {} + if self.read_directory: + directory = self.directory(section_name) + if os.path.isdir(directory): + paths = os.listdir(directory) + for path in sorted(paths): + name, ext = os.path.splitext(path) + abspath = os.path.join(directory, path) + if ext == ".json" and name not in data: + with io.open(abspath, encoding='utf-8') as f: + more_defaults = json.load(f) + recursive_update(data, more_defaults) if os.path.isfile(filename): with io.open(filename, encoding='utf-8') as f: - return json.load(f) - else: - return {} + new_data = json.load(f) + recursive_update(data, new_data) + return data def set(self, section_name, data): """Store the given config data. From 9cae1ba4e92e201856ecdea7a3b00f4d39788750 Mon Sep 17 00:00:00 2001 From: maartenbreddels Date: Thu, 7 Dec 2017 19:50:15 +0100 Subject: [PATCH 2/4] cleanups/simplification after suggestions by @bollwyvl --- traitlets/config/manager.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/traitlets/config/manager.py b/traitlets/config/manager.py index 6a2ff1e0..8d341f91 100644 --- a/traitlets/config/manager.py +++ b/traitlets/config/manager.py @@ -3,6 +3,7 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. import errno +import glob import io import json import os @@ -62,23 +63,15 @@ def get(self, section_name): Returns the data as a dictionary, or an empty dictionary if the file doesn't exist. """ - filename = self.file_name(section_name) - data = {} + paths = [self.file_name(section_name)] if self.read_directory: - directory = self.directory(section_name) - if os.path.isdir(directory): - paths = os.listdir(directory) - for path in sorted(paths): - name, ext = os.path.splitext(path) - abspath = os.path.join(directory, path) - if ext == ".json" and name not in data: - with io.open(abspath, encoding='utf-8') as f: - more_defaults = json.load(f) - recursive_update(data, more_defaults) - if os.path.isfile(filename): - with io.open(filename, encoding='utf-8') as f: - new_data = json.load(f) - recursive_update(data, new_data) + pattern = os.path.join(self.directory(section_name), '*.json') + paths = sorted(glob.glob(pattern)) + paths + data = {} + for path in paths: + if os.path.isfile(path): + with io.open(path, encoding='utf-8') as f: + recursive_update(data, json.load(f)) return data def set(self, section_name, data): From 5cc136587aeec971778d48a7100c82301f7a1244 Mon Sep 17 00:00:00 2001 From: maartenbreddels Date: Thu, 7 Dec 2017 19:59:58 +0100 Subject: [PATCH 3/4] some comments on the order --- traitlets/config/manager.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/traitlets/config/manager.py b/traitlets/config/manager.py index 8d341f91..05f80716 100644 --- a/traitlets/config/manager.py +++ b/traitlets/config/manager.py @@ -66,6 +66,11 @@ def get(self, section_name): paths = [self.file_name(section_name)] if self.read_directory: pattern = os.path.join(self.directory(section_name), '*.json') + # These json files should be processed first so that the + # {section_name}.json take precedence. + # The idea behind this is that installing a Python package may + # put a json file somewhere in the a .d directory, while the + # .json file is probably a user configuration. paths = sorted(glob.glob(pattern)) + paths data = {} for path in paths: From 1a3730201f9d72f5c9baa9e020acbab38564e536 Mon Sep 17 00:00:00 2001 From: maartenbreddels Date: Thu, 7 Dec 2017 20:36:44 +0100 Subject: [PATCH 4/4] test: add unittests for the BaseJSONConfigManager --- traitlets/config/tests/test_manager.py | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 traitlets/config/tests/test_manager.py diff --git a/traitlets/config/tests/test_manager.py b/traitlets/config/tests/test_manager.py new file mode 100644 index 00000000..0ce4f740 --- /dev/null +++ b/traitlets/config/tests/test_manager.py @@ -0,0 +1,34 @@ +import json +import os + +from traitlets.config.manager import BaseJSONConfigManager + + +def test_json(tmpdir): + tmpdir = str(tmpdir) # we're ok with a regular string path + with open(os.path.join(tmpdir, 'foo.json'), 'w') as f: + json.dump(dict(a=1), f) + # also make a foo.d/ directory with multiple json files + os.makedirs(os.path.join(tmpdir, 'foo.d')) + with open(os.path.join(tmpdir, 'foo.d', 'a.json'), 'w') as f: + json.dump(dict(a=2, b=1), f) + with open(os.path.join(tmpdir, 'foo.d', 'b.json'), 'w') as f: + json.dump(dict(a=3, b=2, c=3), f) + manager = BaseJSONConfigManager(config_dir=tmpdir, read_directory=False) + data = manager.get('foo') + assert 'a' in data + assert 'b' not in data + assert 'c' not in data + assert data['a'] == 1 + + manager = BaseJSONConfigManager(config_dir=tmpdir, read_directory=True) + data = manager.get('foo') + assert 'a' in data + assert 'b' in data + assert 'c' in data + # files should be read in order foo.d/a.json foo.d/b.json foo.json + assert data['a'] == 1 + assert data['b'] == 2 + assert data['c'] == 3 + +