From b0a523f34a52301f78ede37b20e26acc1c4754f6 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Sat, 16 Sep 2023 17:12:14 -0400 Subject: [PATCH] [#471][#472] allow save, load, and autoload of configuration --- README.rst | 31 ++++ irods/__init__.py | 20 ++ irods/client_configuration/__init__.py | 173 ++++++++++++++++++ irods/connection.py | 1 - irods/test/data_obj_test.py | 49 ++++- irods/test/helpers.py | 39 +++- ...ving_and_loading_of_settings__issue_471.py | 33 ++++ 7 files changed, 339 insertions(+), 7 deletions(-) create mode 100644 irods/test/modules/test_saving_and_loading_of_settings__issue_471.py diff --git a/README.rst b/README.rst index 10b6ab25..50c991f8 100644 --- a/README.rst +++ b/README.rst @@ -314,6 +314,37 @@ This may be useful for Python programs in which frequent flushing of write updat with descriptors on such objects possibly being held open for indeterminately long lifetimes -- yet the eventual application of those updates prior to the teardown of the Python interpreter is required. +The current value of the setting is global in scope (ie applies to all sessions, whenever created) and is always +consulted for the creation of any data object handle to govern that handle's cleanup behavior. + +Python iRODS Client Settings File +--------------------------------- + +As of v1.1.9, Python iRODS client configuration can be saved in, and loaded from, a settings file. + +If the settings file exists, each of its lines contains (a) a dotted name identifying a particular configuration setting +to be assigned within the PRC, potentially changing its runtime behavior; and (b) the specific value, in Python "repr"-style +format, that should be assigned into it. + +An example follows: + + data_objects.auto_close True + +New dotted names may be created following the example of the one valid example created thus far, +code:`data_objects.auto_close`, initialized in :code:`irods/client_configuration/__init__.py`. Each such name should correspond +to a globally set value which the PRC routinely checks when performing the affected library function. + +The use of a settings file can be indicated, and the path to that file determined, by setting the environment variable: +:code:`PYTHON_IRODSCLIENT_CONFIGURATION_PATH`. If this variable is present but empty, this denotes use of a default settings +file path of :code:~/.python-irodsclient`; if the variable's value is of nonzero length, the value should be an absolute path +to the settings file whose use is desired. Also, if the variable is set, auto-load of settings will be performed, meaning +that the act of importing :code:`irods` or any of its submodules will cause the automatic loading the settings from the +settings file, assuming it exists. (Failure to find the file at the indicated path will be logged as a warning.) + +Settings can also be saved and loaded manually using the save() and load() functions in the :code:`irods.client_configuration` +module. Each of these functions accepts an optional :code:`file` parameter which, if set to a non-empty string, will override +the settings file path currently "in force" (i.e., the CONFIG_DEFAULT_PATH, as optionally overridden by the environment variable +PYTHON_IRODSCLIENT_CONFIGURATION_PATH). Computing and Retrieving Checksums ---------------------------------- diff --git a/irods/__init__.py b/irods/__init__.py index d88d0d45..ef1c1ce4 100644 --- a/irods/__init__.py +++ b/irods/__init__.py @@ -1,6 +1,11 @@ from .version import __version__ import logging +import os + +# This has no effect if basicConfig() was previously called. +logging.basicConfig() + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) gHandler = None @@ -50,3 +55,18 @@ def client_logging(flag=True,handler=None): PAM_AUTH_PLUGIN = 'PAM' PAM_AUTH_SCHEME = PAM_AUTH_PLUGIN.lower() + +DEFAULT_CONFIG_PATH = os.path.expanduser('~/.python_irodsclient') +settings_path_environment_variable = 'PYTHON_IRODSCLIENT_CONFIGURATION_PATH' + +def get_settings_path(): + env_var = os.environ.get(settings_path_environment_variable) + return DEFAULT_CONFIG_PATH if not env_var else env_var + +from . import client_configuration + +client_configuration.preserve_defaults() + +# If the settings path variable is not set in the environment, a value of None is passed, +# and thus no settings file is auto-loaded. +client_configuration.autoload(_file_to_load = os.environ.get(settings_path_environment_variable)) diff --git a/irods/client_configuration/__init__.py b/irods/client_configuration/__init__.py index 0ba0832f..cd495e6b 100644 --- a/irods/client_configuration/__init__.py +++ b/irods/client_configuration/__init__.py @@ -3,16 +3,30 @@ import copy import io import logging +import os import re import sys import types +# Duplicate here for convenience +from .. import DEFAULT_CONFIG_PATH + logger = logging.Logger(__name__) class iRODSConfiguration(object): __slots__ = () def getter(category, setting): + """A programmatic way of allowing the current value of the specified setting to be + given indirectly (through an extra call indirection) as the default value of a parameter. + + Returns a lambda that, when called, will yield the setting's value. In the closure of + that lambda, the Python builtin function globals() is used to access (in a read-only + capacity) the namespace dict of the irods.client_configuration module. + + See the irods.manager.data_object_manager.DataObjectManager.open(...) method signature + for a usage example. + """ return lambda:getattr(globals()[category], setting) # ############################################################################# @@ -48,3 +62,162 @@ def __init__(self): # listed in the __slots__ member of the category class. data_objects = DataObjects() + +def _var_items(root): + if isinstance(root,types.ModuleType): + return [(i,v) for i,v in vars(root).items() + if isinstance(v,iRODSConfiguration)] + if isinstance(root,iRODSConfiguration): + return [(i, getattr(root,i)) for i in root.__slots__] + return [] + +def save(root = None, string='', file = ''): + """Save the current configuration. + + When called simply as save(), this function simply writes all client settings into + a configuration file. + + The 'root' and 'string' parameters are not likely to be overridden when called from an + application. They should usually only vary from the defaults when save() recurses into itself. + However, for due explanation's sake: 'root' specifies at which subtree node to start writing, + None denoting the top level; and 'string' specifies a prefix for the dotted prefix name, + which should be empty for an invocation that references the settings' top level namespace. + Both of these defaults are in effect when calling save() without explicit parameters. + + The configuration file path will normally be the value of DEFAULT_CONFIG_PATH, + but this can be overridden by supplying a non-empty string in the 'file' parameter. + """ + _file = None + auto_close_settings = False + try: + if not file: + from .. import get_settings_path + file = get_settings_path() + if isinstance(file,str): + _file = open(file,'w') + auto_close_settings = True + else: + _file = file # assume file-like object if not a string + if root is None: + root = sys.modules[__name__] + for k,v in _var_items(root): + dotted_string = string + ("." if string else "") + k + if isinstance(v,iRODSConfiguration): + save(root = v, string = dotted_string, file = _file) + else: + print(dotted_string, repr(v), sep='\t\t', file = _file) + return file + finally: + if _file and auto_close_settings: + _file.close() + +def _load_config_line(root, setting, value): + arr = [_.strip() for _ in setting.split('.')] + # Compute the object referred to by the dotted name. + attr = '' + for i in filter(None,arr): + if attr: + root = getattr(root,attr) + attr = i + # Assign into the current setting of the dotted name (effectively .) + # using the loaded value. + if attr: + return setattr(root, attr, ast.literal_eval(value)) + error_message = 'Bad setting: root = {root!r}, setting = {setting!r}, value = {value!r}'.format(**locals()) + raise RuntimeError (error_message) + +# The following regular expression is used to match a configuration file line of the form: +# --------------------------------------------------------------- +# +# key: +# +# value: +# + +_key_value_pattern = re.compile(r'\s*(?P\w+(\.\w+)+)\s+(?P\S.*?)\s*$') + +class _ConfigLoadError: + """ + Exceptions that subclass this type can be thrown by the load() function if + their classes are listed in the failure_modes parameter of that function. + """ + +class NoConfigError(Exception, _ConfigLoadError): pass +class BadConfigError(Exception, _ConfigLoadError): pass + +def load(root = None, file = '', failure_modes = (), logging_level = logging.WARNING): + """Load the current configuration. + + An example of a valid line in a configuration file is this: + + data_objects.auto_close True + + When this function is called without parameters, it reads all client settings from + a configuration file (the path given by DEFAULT_CONFIG_PATH, since file = '' in such + an invocation) and assigns the repr()-style Python value given into the dotted-string + configuration entry given. + + The 'file' parameter, when set to a non-empty string, provides an override for + the config-file path default. + + As with save(), 'root' refers to the starting location in the settings tree, with + a value of None denoting the top tree node (ie the namespace containing *all* settings). + There are as yet no imagined use-cases for an application developer to pass in an + explicit 'root' override. + + 'failure_modes' is an iterable containing desired exception types to be thrown if, + for example, the configuration file is missing (NoConfigError) or contains an improperly + formatted line (BadConfigError). + + 'logging_level' governs the internally logged messages and can be used to e.g. quiet the + call's logging output. + """ + def _existing_config(path): + if os.path.isfile(path): + return open(path,'r') + message = 'Config file not available at %r' % (path,) + logging.getLogger(__name__).log(logging_level, message) + if NoConfigError in failure_modes: + raise NoConfigError(message) + return io.StringIO() + + _file = None + try: + if not file: + from .. import get_settings_path + file = get_settings_path() + + _file = _existing_config(file) + + if root is None: + root = sys.modules[__name__] + + for line_number, line in enumerate(_file.readlines()): + line = line.strip() + match = _key_value_pattern.match(line) + if not match: + if line != '': + # Log only the invalid lines that contain non-whitespace characters. + message = 'Invalid configuration format at line %d: %r' % (line_number+1, line) + logging.getLogger(__name__).log(logging_level, message) + if BadConfigError in failure_modes: + raise BadConfigError(message) + continue + _load_config_line(root, match.group('key'), match.group('value')) + finally: + if _file: + _file.close() + +default_config_dict = {} + +def preserve_defaults(): + default_config_dict.update((k,copy.deepcopy(v)) for k,v in globals().items() if isinstance(v,iRODSConfiguration)) + +def autoload(_file_to_load): + if _file_to_load is not None: + load(file = _file_to_load) + +def new_default_config(): + module = types.ModuleType('_') + module.__dict__.update(default_config_dict) + return module diff --git a/irods/connection.py b/irods/connection.py index f395b165..c7aa8942 100644 --- a/irods/connection.py +++ b/irods/connection.py @@ -12,7 +12,6 @@ from ast import literal_eval as safe_eval import re - PAM_PW_ESC_PATTERN = re.compile(r'([@=&;])') diff --git a/irods/test/data_obj_test.py b/irods/test/data_obj_test.py index 20c03ca6..fa34748a 100644 --- a/irods/test/data_obj_test.py +++ b/irods/test/data_obj_test.py @@ -41,11 +41,12 @@ def is_localhost_synonym(name): import irods.test.helpers as helpers import irods.test.modules as test_modules import irods.keywords as kw +import irods.client_configuration as config from irods.manager import data_object_manager from irods.message import RErrorStack from irods.message import ( ET, XML_Parser_Type, default_XML_parser, current_XML_parser ) from datetime import datetime -from tempfile import NamedTemporaryFile, mktemp +from tempfile import NamedTemporaryFile, gettempdir from irods.test.helpers import (unique_name, my_function_name) from irods.ticket import Ticket import irods.parallel @@ -1911,6 +1912,52 @@ def test_data_objects_auto_close_on_function_exit__issue_456(self): data_object_path, expected_content = test_module.test(return_locals = ('name','expected_content')) self._auto_close_test(data_object_path, expected_content) + @unittest.skipIf(helpers.configuration_file_exists(),"test would overwrite pre-existing configuration.") + def test_settings_save_and_autoload__issue_471(self): + import irods.test.modules.test_saving_and_loading_of_settings__issue_471 as test_module + truth = int(time.time()) + test_output = test_module.test(truth) + self.assertEqual(test_output, str(truth)) + + def test_settings_load_and_save_471(self): + from irods import settings_path_environment_variable, get_settings_path, DEFAULT_CONFIG_PATH + settings_path = get_settings_path() + with helpers.file_backed_up(settings_path, require_that_file_exists = False): + + RANDOM_VALUE=int(time.time()) + config.data_objects.auto_close = RANDOM_VALUE + + # Create empty settings file. + with open(settings_path,'w'): + pass + + # For "silent" loading. + load_logging_options = {'logging_level':logging.DEBUG} + + config.load(**load_logging_options) + + # Load from empty settings should change nothing. + self.assertTrue(config.data_objects.auto_close, RANDOM_VALUE) + + os.unlink(settings_path) + config.load(**load_logging_options) + # Load from nonexistent settings file should change nothing. + self.assertTrue(config.data_objects.auto_close, RANDOM_VALUE) + + with helpers.environment_variable_backed_up(settings_path_environment_variable): + os.environ.pop(settings_path_environment_variable,None) + tmp_path = os.path.join(gettempdir(),'.prc') + for i, test_path in enumerate([None, '', tmp_path]): + if test_path is not None: + os.environ[settings_path_environment_variable] = test_path + # Check that load and save work as expected. + config.data_objects.auto_close = RANDOM_VALUE - i - 1 + saved_path = config.save() + # File path should be as expected. + self.assertEqual(saved_path, (DEFAULT_CONFIG_PATH if not test_path else test_path)) + config.data_objects.auto_close = RANDOM_VALUE + config.load(**load_logging_options) + self.assertTrue(config.data_objects.auto_close, RANDOM_VALUE - i - 1) if __name__ == '__main__': # let the tests find the parent irods lib diff --git a/irods/test/helpers.py b/irods/test/helpers.py index 0fbfd83c..b4254fc2 100644 --- a/irods/test/helpers.py +++ b/irods/test/helpers.py @@ -7,6 +7,7 @@ import shutil import hashlib import base64 +import logging import math import socket import inspect @@ -15,7 +16,7 @@ import datetime import json import sys -import logging +import irods.client_configuration as config from irods.session import iRODSSession from irods.message import (iRODSMessage, IRODS_VERSION) from irods.password_obfuscation import encode @@ -58,6 +59,14 @@ def __del__(self): self.admin.users.remove(username) +def configuration_file_exists(): + try: + config.load(failure_modes = (config.NoConfigError,), logging_level = logging.DEBUG) + except config.NoConfigError: + return False + return True + + class StopTestsException(Exception): def __init__(self,*args,**kwargs): @@ -307,14 +316,34 @@ def remove_unused_metadata(session): @contextlib.contextmanager -def file_backed_up(filename): - with tempfile.NamedTemporaryFile(prefix=os.path.basename(filename)) as f: - shutil.copyfile(filename, f.name) +def file_backed_up(filename, require_that_file_exists = True): + _basename = os.path.basename(filename) if os.path.exists(filename) else None + if _basename is None and require_that_file_exists: + err = RuntimeError("Attempted to back up a file which doesn't exist: %r" % (filename,)) + raise err + with tempfile.NamedTemporaryFile(prefix=('tmp' if not _basename else _basename)) as f: try: + if _basename is not None: + shutil.copyfile(filename, f.name) yield filename finally: - shutil.copyfile(f.name, filename) + if _basename is None: + # Restore the condition of the file being absent. + if os.path.exists(filename): + os.unlink(filename) + else: + # Restore the file's contents as they were originally. + shutil.copyfile(f.name, filename) +@contextlib.contextmanager +def environment_variable_backed_up(var): + old_value = os.environ.get(var) + try: + yield var + finally: + os.environ.pop(var,None) + if old_value is not None: + os.environ[var] = old_value def irods_session_host_local (sess): return socket.gethostbyname(sess.host) == \ diff --git a/irods/test/modules/test_saving_and_loading_of_settings__issue_471.py b/irods/test/modules/test_saving_and_loading_of_settings__issue_471.py new file mode 100644 index 00000000..cab8acfe --- /dev/null +++ b/irods/test/modules/test_saving_and_loading_of_settings__issue_471.py @@ -0,0 +1,33 @@ +from __future__ import print_function +import sys +import os +import subprocess + +import irods +import irods.client_configuration as config +from irods.test import modules as test_modules +from irods.test.modules.test_auto_close_of_data_objects__issue_456 import auto_close_data_objects + +def test(truth_value): + """Temporarily sets the data object auto-close setting true and launches a new interpreter + to ensure the setting is auto-loaded. + """ + program = os.path.join(test_modules.__path__[0], os.path.basename(__file__)) + try: + os.putenv(irods.settings_path_environment_variable,'') + with auto_close_data_objects(truth_value): + config.save() + # Call into this same module as a command. This will cause another Python interpreter to start + # up in a separate process and execute the function run_as_process() to test the saved setting. + # We return from this function the output of that process, stripped of whitespace. + process = subprocess.Popen([sys.executable, program], stdout=subprocess.PIPE) + return process.communicate()[0].decode().strip() + finally: + os.unsetenv(irods.settings_path_environment_variable) + os.unlink(config.DEFAULT_CONFIG_PATH) + +def run_as_process(): + print (config.data_objects.auto_close) + +if __name__ == '__main__': + run_as_process()