Split runtime from static configuration and use it for request context.

mig/server/grid_notify.py

@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
@@ -20,7 +20,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 # -- END_HEADER ---
 #
@@ -130,12 +130,10 @@
         notify_message = "Found %s new events since: %s\n\n" \
             % (total_events, timestr) \
             + notify_message
-        status = send_email(
+        status = send_email(configuration,
             recipient,
             subject,
-            notify_message,
-            logger,
-            configuration)
+            notify_message)
         if status:
             logger.info("Send email with %s events to: %s"
                         % (total_events, recipient))
@@ -341,7 +339,7 @@
         print(err_msg)
         sys.exit(1)
 
     print('''This is the MiG system notify daemon which notify users about system events.
 
 Set the MIG_CONF environment to the server configuration path
 unless it is available in mig/server/MiGserver.conf

mig/shared/conf.py

@@ -20,7 +20,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 # -- END_HEADER ---
 #
@@ -32,17 +32,59 @@
 import os
 import sys
 
+from mig.shared.configuration import Configuration
 from mig.shared.defaults import MIG_ENV
 from mig.shared.fileio import unpickle
 
 
-def get_configuration_object(config_file=None, skip_log=False,
-                             disable_auth_log=False):
+class RuntimeConfiguration:
+    """A proxy object to be passed in-place of a Configuration which can be
+    extended with information relevant only at runtime.
+
+    Given Configuration objects are threaded into and through almost all
+    the necessary codepaths to make this information available, they are an
+    attractive place to put this - but a Configuration is currently loaded
+    from static per-site data.
+
+    Resolve this dichotomy with this class - a Configuration instance will
+    continue to represent the static data while an object that proxies its
+    attributes and thus is entirely drop-in compatible with it can be handed
+    to callers without being mixed in with the statuc data.
+    """
+
+    def __init__(self, configuration):
+        object.__setattr__(self, '_configuration', configuration)
+        object.__setattr__(self, '_context', {})
+
+    def __delattr__(self, attr):
+        return delattr(self._configuration, attr)
+
+    def __getattr__(self, attr):
+        return getattr(self._configuration, attr)
+
+    def __setattr__(self, attr, value):
+        return setattr(self._configuration, attr, value)
+
+    def context_get(self, context_key):
+        return self._context.get(context_key, None)
+
+    def context_set(self, context_key, context_value):
+        self._context[context_key] = context_value
+
+    @classmethod
+    def is_runtime_configuration(cls, obj):
+        """Is the given object a runtime configuration."""
+        return isinstance(obj, cls)
+
+
+def get_configuration_object(config_file=None,
+                             skip_log=False,
+                             disable_auth_log=False,
+                             _raw=False):
     """Simple helper to call the general configuration init. Optional skip_log
     and disable_auth_log arguments are passed on to allow skipping the default
     log initialization and disabling auth log for unit tests.
     """
-    from mig.shared.configuration import Configuration
     if config_file:
         _config_file = config_file
     elif os.environ.get('MIG_CONF', None):
@@ -65,7 +107,11 @@
 
     configuration = Configuration(_config_file, False, skip_log,
                                   disable_auth_log)
-    return configuration
+
+    if _raw:
+        return configuration
+
+    return RuntimeConfiguration(configuration)
 
 
 def get_resource_configuration(resource_home, unique_resource_name,

mig/shared/configuration.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -20,7 +20,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 # -- END_HEADER ---
 #
@@ -83,6 +83,7 @@
     'skip_log',
     'verbose',
     'logger',
+    '_context',
 ])
 
 
@@ -170,7 +171,7 @@
                     #    "reading conf content from file in %r" % cache)
                     content = read_file(path, logger)
                     if cache and content:
                         # logger.debug("caching conf content salt in %r" % cache)
                         write_file(content, cache, logger)
                 if not content:
                     logger.warning("salt file not found or empty: %s" % path)
@@ -244,7 +245,7 @@
         'server_home': '~/state/server_home/',
         'webserver_home': '~/state/webserver_home/',
         'sessid_to_mrsl_link_home': '~/state/sessid_to_mrsl_link_home/',
         'sessid_to_jupyter_mount_link_home': '~/state/sessid_to_jupyter_mount_link_home/',
         'mig_system_files': '~/state/mig_system_files/',
         'mig_system_storage': '~/state/mig_system_storage',
         'mig_system_run': '~/state/mig_system_run/',
@@ -385,7 +386,7 @@
         'architectures': 'X86 AMD64 IA64 SPARC SPARC64 ITANIUM SUN4U SPARC-T1',
         'scriptlanguages': 'sh python java',
         'jobtypes': 'batch interactive bulk all',
         'lrmstypes': 'Native Native-execution-leader Batch Batch-execution-leader',
     }
     scheduler_section = {'algorithm': 'FairFit',
                          'expire_after': '99999999999',
@@ -2921,6 +2922,10 @@
                     peerfile)
         return peers_dict
 
+    @classmethod
+    def create(cls, *args, **kwargs):
+        return cls(*args, **kwargs)
+
     @staticmethod
     def is_configuration_like(obj):
         """Does the given object quack like a MiG Configuration."""

mig/shared/functionality/autocreate.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -740,8 +740,7 @@
         logger.info('Send email: to: %s, header: %s, smtp_server: %s'
                     % (email, email_header, smtp_server))
         logger.debug('email body:  %s' % email_msg)
-        if not send_email(email, email_header, email_msg, logger,
-                          configuration):
+        if not send_email(configuration, email, email_header, email_msg):
             output_objects.append({
                 'object_type': 'error_text', 'text': """An error occurred trying
 to send your account welcome email. Please contact support (%s) and include the

mig/shared/functionality/extcertaction.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -316,8 +316,7 @@
 
     logger.info('Sending email: to: %s, header: %s, msg: %s, smtp_server: %s'
                 % (admin_email, email_header, email_msg, smtp_server))
-    if not send_email(admin_email, email_header, email_msg, logger,
-                      configuration):
+    if not send_email(configuration, admin_email, email_header, email_msg):
         output_objects.append({'object_type': 'error_text', 'text':
                                """An error occurred trying to inform the site
 admins about your request for existing certificate sign up. Please contact %s site

mig/shared/functionality/extoidaction.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -282,8 +282,7 @@
 
     logger.info('Sending email: to: %s, header: %s, msg: %s, smtp_server: %s'
                 % (admin_email, email_header, email_msg, smtp_server))
-    if not send_email(admin_email, email_header, email_msg, logger,
-                      configuration):
+    if not send_email(configuration, admin_email, email_header, email_msg):
         output_objects.append({'object_type': 'error_text', 'text':
                                """An error occurred trying to inform the site
 admins about your request for OpenID account access. Please contact %s site

mig/shared/functionality/peersaction.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -309,8 +309,7 @@
                     logger.info('Sending invitation: to: %s, header: %s, msg: %s, smtp_server: %s'
                                 % (peer_email, email_header, email_msg,
                                    smtp_server))
-                    if send_email(peer_email, email_header, email_msg, logger,
-                                  configuration):
+                    if send_email(configuration, peer_email, email_header, email_msg):
                         succeeded.append(peer_email)
                     else:
                         failed.append(peer_email)
@@ -369,8 +368,7 @@
 
     logger.info('Sending email: to: %s, header: %s, msg: %s, smtp_server: %s'
                 % (admin_email, email_header, email_msg, smtp_server))
-    if not send_email(admin_email, email_header, email_msg, logger,
-                      configuration):
+    if not send_email(configuration, admin_email, email_header, email_msg):
         output_objects.append({'object_type': 'error_text', 'text': '''
 An error occurred trying to send the email about your %s peers to the site
 administrators. Please contact %s site support at %s or manually inform the

mig/shared/functionality/reqcertaction.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -346,8 +346,7 @@
 
     logger.info('Sending email: to: %s, header: %s, msg: %s, smtp_server: %s'
                 % (admin_email, email_header, email_msg, smtp_server))
-    if not send_email(admin_email, email_header, email_msg, logger,
-                      configuration):
+    if not send_email(configuration, admin_email, email_header, email_msg):
         output_objects.append({'object_type': 'error_text', 'text':
                                """An error occurred trying to inform the site
 admins about your request for certificate sign up. Please contact %s site

mig/shared/functionality/reqoidaction.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -357,8 +357,7 @@
 
     logger.info('Sending email: to: %s, header: %s, msg: %s, smtp_server: %s'
                 % (admin_email, email_header, email_msg, smtp_server))
-    if not send_email(admin_email, email_header, email_msg, logger,
-                      configuration):
+    if not send_email(configuration, admin_email, email_header, email_msg):
         output_objects.append({'object_type': 'error_text', 'text':
                                """An error occurred trying to inform the site
 admins about your request for OpenID account access. Please contact %s site

mig/shared/functionality/reqpwresetaction.py

@@ -271,8 +271,7 @@ def main(client_id, user_arguments_dict):
 
         logger.info('Sending email: to: %s, header: %s, msg: %s, smtp_server: %s'
                     % (email_to, email_header, email_msg, smtp_server))
-        if not send_email(email_to, email_header, email_msg, logger,
-                          configuration):
+        if not send_email(configuration, email_to, email_header, email_msg):
             output_objects.append({'object_type': 'error_text', 'text':
                                    """An error occurred trying to send the email
 for an account %s password reset request. Please contact %s site support at %s

mig/shared/gdp/base.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -683,12 +683,10 @@
 
 Attached you'll find the details registered in relation to the operation.
         """ % mail_fill
-        status = send_email(
+        status = send_email(configuration,
             recipients,
             subject,
             message,
-            _logger,
-            configuration,
             files=[pdf_filepath],
         )
         if status:

mig/shared/notification.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -52,6 +52,7 @@
 
 from mig.shared.base import force_utf8, generate_https_urls, extract_field, \
     canonical_user_with_peers, cert_field_map, get_site_base_url
+from mig.shared.conf import RuntimeConfiguration
 from mig.shared.defaults import email_keyword_list, job_output_dir, \
     transfer_output_dir, keyword_auto, cert_auto_extend_days, \
     oid_auto_extend_days
@@ -561,11 +562,35 @@
 
 
 def send_email(
+        configuration,
+        recipients,
+        subject,
+        message,
+        files=[],
+        custom_sender=None):
+    """Send email to recipients via the actively configured method."""
+
+    assert RuntimeConfiguration.is_runtime_configuration(configuration)
+
+    notifier = configuration.context_get('notifier')
+    if not notifier:
+        notifier = Notifier(configuration)
+        configuration.context_set('notifier', notifier)
+
+    return notifier.send_email(
+        recipients,
+        subject,
+        message,
+        files=files,
+        custom_sender=custom_sender,
+    )
+
+
+def direct_send_email(
+    configuration,
     recipients,
     subject,
     message,
-    logger,
-    configuration,
     files=[],
     custom_sender=None
 ):
@@ -589,13 +614,14 @@
     """
 
     _logger = configuration.logger
+
     gpg_sign = False
     if configuration.site_gpg_passphrase is not None:
         if gnupg is None:
-            logger.warning("the gnupg module is required for gpg signing")
+            _logger.warning("the gnupg module is required for gpg signing")
         else:
             gpg_sign = True
-            logger.debug("enabling automatic gpg signing of email")
+            _logger.debug("enabling automatic gpg signing of email")
 
     if recipients.find(', ') > -1:
         recipients_list = recipients.split(', ')
@@ -631,7 +657,7 @@
         basemsg = MIMEText(force_utf8(message), "plain", "utf8")
         mime_msg.attach(basemsg)
         if gpg_sign:
-            logger.info("signing message with gpg")
+            _logger.info("signing message with gpg")
             gpg = gnupg.GPG()
             basetext = basemsg.as_string().replace('\n', '\r\n')
             signature = gpg.sign(basetext, detach=True,
@@ -643,7 +669,7 @@
                 msg_sig.set_payload("%s" % signature)
                 mime_msg.attach(msg_sig)
             else:
-                logger.warning("failed to create gnupg signature")
+                _logger.warning("failed to create gnupg signature")
 
         for name in files:
             part = MIMEBase('application', "octet-stream")
@@ -652,7 +678,7 @@
             part.add_header('Content-Disposition',
                             'attachment', filename=os.path.basename(name))
             mime_msg.attach(part)
-        logger.debug('sending email from %s to %s:\n%s' %
+        _logger.debug('sending email from %s to %s:\n%s' %
                      (from_email, recipients, mime_msg.as_string()))
         server = smtplib.SMTP(configuration.smtp_server)
         server.set_debuglevel(0)
@@ -664,10 +690,10 @@
                             % errors)
             return False
         else:
-            logger.debug('Email was sent to %s' % recipients)
+            _logger.debug('Email was sent to %s' % recipients)
             return True
     except Exception as err:
-        logger.error('Sending email to %s through %s failed!: %s'
+        _logger.error('Sending email to %s through %s failed!: %s'
                      % (recipients, configuration.smtp_server, err))
         return False
 
@@ -819,8 +845,8 @@
 
                     continue
 
-                if send_email(single_dest, header, message, logger,
-                              configuration, custom_sender=email_sender):
+                if send_email(configuration, single_dest, header, message,
+                              custom_sender=email_sender):
                     logger.info('email sent to %s telling that %s %s'
                                 % (single_dest, jobid, status))
                 else:
@@ -906,7 +932,7 @@
         os.path.basename(pending_file),
     )
 
-    status = send_email(recipients, subject, txt, logger, configuration)
+    status = send_email(configuration, recipients, subject, txt)
     if status:
         msg += '\nEmail was sent to admins'
     else:
@@ -961,3 +987,13 @@
     return send_message_to_grid_notify(pickled_notification,
                                        configuration.logger,
                                        configuration)
+
+
+class Notifier:
+    def __init__(self, configuration):
+        self.configuration = configuration
+
+    def send_email(self, recipients, subject, message, files=[], custom_sender=None):
+        return direct_send_email(self.configuration, recipients, subject, message,
+                                                        files=[],
+                                                        custom_sender=None)

tests/fixture/mig_shared_configuration--new.json

@@ -29,7 +29,6 @@
   "ca_user": "mig-ca",
   "certs_path": "/some/place/certs",
   "cloud_services": [],
-  "config_file": null,
   "cputime_for_empty_jobs": 0,
   "default_page": [
     ""
@@ -104,14 +103,12 @@
   ],
   "log_dir": "",
   "logfile": "",
-  "logger": null,
   "logger_obj": null,
   "loglevel": "",
   "lrmstypes": [],
   "mig_code_base": "",
   "mig_path": "/some/place/mig",
   "mig_server_home": "",
-  "mig_server_id": null,
   "mig_system_files": "",
   "mig_system_run": "",
   "mig_system_storage": "",

tests/support/__init__.py

@@ -48,6 +48,7 @@
 from tests.support.suppconst import MIG_BASE, TEST_BASE, \
     TEST_DATA_DIR, TEST_OUTPUT_DIR, ENVHELP_OUTPUT_DIR
 from tests.support.usersupp import UserAssertMixin
+import tests.support.fakes as fakes
 
 from tests.support._env import MIG_ENV, PY2
 
@@ -212,11 +213,12 @@ def _make_configuration_instance(testcase, configuration_to_make):
         if configuration_to_make == 'fakeconfig':
             return FakeConfiguration(logger=testcase.logger)
         elif configuration_to_make == 'testconfig':
-            from mig.shared.conf import get_configuration_object
-            configuration = get_configuration_object(skip_log=True, 
-                                                     disable_auth_log=True)
+            from mig.shared.conf import get_configuration_object, RuntimeConfiguration
+            configuration = get_configuration_object(skip_log=True,
+                                                     disable_auth_log=True,
+                                                     _raw=True)
             configuration.logger = testcase.logger
-            return configuration
+            return RuntimeConfiguration(configuration)
         else:
             raise AssertionError(
                 "MigTestCase: unknown configuration %r" % (configuration_to_make,))
@@ -250,6 +252,8 @@ def configuration(self):
 
         self._configuration = configuration_instance
 
+        fakes.instrument_test_case(self, self._configuration)
+
         return configuration_instance
 
     @property