Sweep the tree for everything conditional on python 2.

mig/server/grid_events.py

@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
@@ -20,7 +20,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 # -- END_HEADER ---
 #
@@ -73,7 +73,7 @@
     from mig.shared.logger import daemon_logger, register_hangup_handler
     from mig.shared.safeinput import PARAM_START, PARAM_STOP, PARAM_JUMP
     from mig.shared.serial import load
     from mig.shared.vgrid import vgrid_valid_entities, vgrid_add_workflow_jobs, \
         JOB_ID, JOB_CLIENT
     from mig.shared.vgridaccess import check_vgrid_access
     from mig.shared.workflows import get_wp_map, CONF
@@ -517,7 +517,7 @@
 
         if state == 'created':
 
             # logger.debug('(%s) Updating rule monitor for src_path: %s, event: %s'
             #              % (pid, src_path, state))
 
             print('(%s) Updating rule monitor for src_path: %s, event: %s'
@@ -538,10 +538,10 @@
                     # Fire 'modified' events for all dirs and files in subpath
                     # to ensure that all rule files are loaded
 
-                    for ent in scandir(src_path):
+                    for ent in os.scandir(src_path):
                         if ent.is_dir(follow_symlinks=True):
 
                             # logger.debug('(%s) Dispatch DirCreatedEvent for: %s'
                             #         % (pid, ent.path))
 
                             shared_state['rule_handler'].dispatch(
@@ -549,14 +549,14 @@
                         elif ent.path.find(configuration.vgrid_triggers) \
                                 > -1:
 
                             # logger.debug('(%s) Dispatch FileCreatedEvent for: %s'
                             #         % (pid, ent.path))
 
                             shared_state['rule_handler'].dispatch(
                                 FileCreatedEvent(ent.path))
 
                 # else:
                 #    logger.debug('(%s) rule_monitor watch already exists for: %s'
                 #                  % (pid, src_path))
         # else:
         #    logger.debug('(%s) unhandled event: %s for: %s' % (pid,
@@ -592,7 +592,7 @@
             except Exception as exc:
                 new_rules = []
                 if state != 'deleted':
                     logger.error('(%s) failed to load event handler rules from %s (%s)'
                                  % (pid, src_path, exc))
 
             # logger.debug("(%s) loaded new rules from '%s':\n%s" % (pid,
@@ -604,7 +604,7 @@
             # NOTE: we need to iterate over a copy of keys for in-place edits
             for target_path in list(all_rules):
                 all_rules[target_path] = [i for i in
                                           all_rules[target_path] if i['vgrid_name']
                                           != vgrid_name]
                 remain_rules = [i for i in all_rules[target_path]
                                 if i['vgrid_name'] != vgrid_name]
@@ -961,7 +961,7 @@
                     # created.
                     if pattern['parameterize_over']:
                         all_values = []
                         for (var, sweep) in pattern['parameterize_over'].items():
 
                             start = float(sweep[PARAM_START])
                             stop = float(sweep[PARAM_STOP])
@@ -1123,7 +1123,7 @@
                 dir_cache[vgrid_name] = {}
             vgrid_dir_cache = dir_cache[vgrid_name]
 
             # logger.debug('(%s) Updating file monitor for src_path: %s, event: %s'
             #              % (pid, src_path, state))
 
             if os.path.exists(src_path) and os.path.isdir(src_path):
@@ -1139,7 +1139,7 @@
                     # For create this occurs by eg. mkdir -p 'path/subpath/subpath2'
                     # or 'cp -rf'
 
-                    for ent in scandir(src_path):
+                    for ent in os.scandir(src_path):
                         if ent.is_dir(follow_symlinks=True):
                             vgrid_sub_path = strip_base_dirs(ent.path)
 
@@ -1491,7 +1491,7 @@
 
                 # Traverse dirs for subdirs created since last run
 
-                for ent in scandir(vgrid_files_path):
+                for ent in os.scandir(vgrid_files_path):
                     if ent.is_dir(follow_symlinks=True):
                         vgrid_sub_path = strip_base_dirs(ent.path)
                         # Force utf8 everywhere to avoid encoding issues
@@ -1839,7 +1839,7 @@
                                          configuration.vgrid_triggers)
                 all_trigger_rules.append(rule_path)
     else:
-        for ent in scandir(vgrid_home):
+        for ent in os.scandir(vgrid_home):
             if configuration.vgrid_triggers in ent.name:
                 rule_path = ent.path
                 all_trigger_rules.append(rule_path)
@@ -1981,7 +1981,7 @@
 
     # Each top vgrid gets is own process
 
-    for ent in scandir(configuration.vgrid_home):
+    for ent in os.scandir(configuration.vgrid_home):
         vgrid_files_path = os.path.join(configuration.vgrid_files_home,
                                         ent.name)
         if os.path.isdir(ent.path) and os.path.isdir(vgrid_files_path):

mig/shared/base.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -736,23 +736,13 @@
     https://docs.python.org/3/library/sys.html#sys.argv
     for further details.
     """
-    if sys.version_info[0] >= 3:
-        return [os.fsencode(arg) for arg in argv]
-    else:
-        return argv
+    return [os.fsencode(arg) for arg in argv]
 
 
 def NativeStringIO(initial_value=''):
-    """Mock StringIO pseudo-class to create a StringIO matching the native
-    string coding form. That is a BytesIO with utf8 on python 2 and unicode
-    StringIO otherwise. Optional string helpers are automatically converted
-    accordingly.
+    """Pseudo-class wrapper to return a StringIO. This is a unicode StringIO.
     """
-    if sys.version_info[0] >= 3:
-        return io.StringIO(initial_value)
-    else:
-        from StringIO import StringIO
-        return StringIO(initial_value)
+    return io.StringIO(initial_value)
 
 
 def DefaultStringIO(initial_value=''):

mig/shared/cgiscriptstub.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -117,9 +117,8 @@
         #logger.debug("flush stdout")
         sys.stdout.flush()
         #logger.debug("write content: %s" % [output[:64], '..', output[-64:]])
-        # NOTE: always output native strings to stdout but use raw buffer
-        #       for byte output on py3 as explained above.
-        if sys.version_info[0] < 3 or is_default_str_coding(output):
+        # NOTE: use raw buffer for byte output as explained above
+        if is_default_str_coding(output):
             sys.stdout.write(output)
         else:
             sys.stdout.buffer.write(output)

mig/shared/compat.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
 #
 # --- BEGIN_HEADER ---
@@ -36,29 +36,10 @@
 import codecs
 import io
 import sys
-# NOTE: StringIO is only available in python2
-try:
-    import StringIO
-except ImportError:
-    StringIO = None
 
-PY2 = sys.version_info[0] < 3
 _TYPE_UNICODE = type(u"")
 
 
-if PY2:
-    class SimpleNamespace(dict):
-        """Bare minimum SimpleNamespace for Python 2."""
-
-        def __getattribute__(self, name):
-            if name == '__dict__':
-                return dict(**self)
-
-            return self[name]
-else:
-    from types import SimpleNamespace
-
-
 def _is_unicode(val):
     """Return boolean indicating if the value is a unicode string.
 
@@ -72,34 +53,9 @@
     """Given a supplied input which can be either a string or bytes
     return a representation providing string operations while ensuring that
     its contents represent a valid series of textual characters.
-
-    Arrange identical operation across python 2 and 3 - specifically,
-    the presence of invalid UTF-8 bytes (thus the input not being a
-    valid textual string) will trigger a UnicodeDecodeError on PY3.
-    Force the same to occur on PY2.
     """
-    if PY2:
-        # Simulate decoding done by PY3 to trigger identical exceptions
-        # note the use of a forced "utf8" encoding value: this function
-        # is generally used to wrap, for example, substitutions of values
-        # into strings that are defined in the source code. In Python 3
-        # these are mandated to be UTF-8, and thus decoding as "utf8" is
-        # what will be attempted on supplied input. Match it.
-        textual_output = codecs.encode(string_or_bytes, 'utf8')
-    elif not _is_unicode(string_or_bytes):
+    if not _is_unicode(string_or_bytes):
         textual_output = str(string_or_bytes, 'utf8')
     else:
         textual_output = string_or_bytes
     return textual_output
-
-
-def NativeStringIO(initial_value=''):
-    """Mock StringIO pseudo-class to create a StringIO matching the native
-    string coding form. That is a BytesIO with utf8 on python 2 and unicode
-    StringIO otherwise. Optional string helpers are automatically converted
-    accordingly.
-    """
-    if PY2 and StringIO is not None:
-        return StringIO.StringIO(initial_value)
-    else:
-        return io.StringIO(initial_value)

mig/shared/configuration.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -34,6 +34,7 @@
 from builtins import object
 
 import base64
+from configparser import ConfigParser
 import copy
 import datetime
 import functools
@@ -50,13 +51,6 @@
 standard_library.install_aliases()
 
 
-# NOTE: should be handled by future aliases but fails in PAM C extension
-if sys.version_info[0] < 3:
-    # NOTE: always use native py2 version here - new one causes unicode mess
-    from ConfigParser import ConfigParser
-else:
-    from configparser import ConfigParser
-
 # NOTE: protect migrid import from autopep8 reordering
 try:
     from mig.shared.base import force_native_str

mig/shared/defaults.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -37,17 +37,8 @@
 MIG_BASE = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..'))
 MIG_ENV = os.getenv('MIG_ENV', 'default')
 
-# NOTE: python3 switched strings to use unicode by default in contrast to bytes
-#       in python2. File systems remain with utf8 however so we need to
-#       carefully handle a lot of cases of either encoding to utf8 or decoding
-#       to unicode depending on the python used.
-#       Please refer to the helpers in shared.base for actual handling of it.
-if sys.version_info[0] >= 3:
-    default_str_coding = 'unicode'
-    default_fs_coding = 'utf8'
-else:
-    default_str_coding = 'utf8'
-    default_fs_coding = 'utf8'
+default_str_coding = 'unicode'
+default_fs_coding = 'utf8'
 
 CODING_KINDS = (STR_KIND, FS_KIND) = ('__STR__', '__FS__')
 

mig/shared/output.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -651,7 +651,7 @@
         lines = [status_line] + lines
 
     # NOTE: careful handling required for binary on python3+
-    if sys.version_info[0] > 2 and binary_output:
+    if binary_output:
         return b''.join(lines)
     else:
         return ''.join(lines)
@@ -2646,7 +2646,7 @@
                                          user_widgets))
 
     # NOTE: careful handling required for binary on python3+
-    if sys.version_info[0] > 2 and binary_output:
+    if binary_output:
         return b''.join(lines)
     else:
         return '\n'.join(lines)

mig/shared/ssh.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -44,9 +44,8 @@
     # Paramiko not available - imported fom griddaemons so fail gracefully
     paramiko = None
 
-from mig.shared.base import client_id_dir, force_utf8
+from mig.shared.base import client_id_dir, force_utf8, NativeStringIO
 from mig.shared.conf import get_resource_exe, get_configuration_object
-from mig.shared.compat import NativeStringIO
 from mig.shared.defaults import ssh_conf_dir
 from mig.shared.safeeval import subprocess_popen, subprocess_pipe
 

mig/shared/url.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
@@ -42,20 +42,9 @@
 import base64
 import os
 import sys
-
-# NOTE: moved to urllib.parse in python3 and are re-exposed with future.
-#       Other modules should import helpers from here for consistency.
-# TODO: handle the unicode returned by python3 and future versions!
-#       Perhaps switch to suggested "easiest option" from
-#       http://python-future.org/compatible_idioms.html#urllib-module
-#       once we have unicode/bytecode mix-up sorted out.
-if sys.version_info[0] >= 3:
-    from urllib.parse import quote, unquote, urlencode, parse_qs, parse_qsl, \
-        urlsplit, urlparse, urljoin
-    from urllib.request import urlopen
-else:
-    from urllib import quote, unquote, urlencode, urlopen
-    from urlparse import parse_qs, parse_qsl, urlsplit, urlparse, urljoin
+from urllib.parse import quote, unquote, urlencode, parse_qs, parse_qsl, \
+    urlsplit, urlparse, urljoin
+from urllib.request import urlopen
 
 try:
     from mig.shared.base import force_utf8, force_native_str

mig/unittest/testcore.py

@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
@@ -38,7 +38,7 @@
 
 sys.path.append(os.path.realpath(
     os.path.join(os.path.dirname(__file__), "../..")))
-from tests.support import MIG_BASE, PY2, is_path_within
+from tests.support import MIG_BASE, is_path_within
 
 from mig.shared.base import client_id_dir, client_dir_id, get_short_id, \
     invisible_path, allow_script, brief_list

mig/wsgi-bin/migwsgi.py

@@ -249,11 +249,6 @@ def application(environ, start_response, configuration=None,
     #      (sys.version_info, ), file=environ['wsgi.errors'])
     # print("DEBUG: path %s" % sys.path, file=environ['wsgi.errors'])
 
-    # NOTE: redirect stdout to stderr in python 2 only. It breaks logger in 3
-    #       and stdout redirection apparently is already handled there.
-    if sys.version_info[0] < 3:
-        sys.stdout = sys.stderr
-
     if configuration is None:
         configuration = get_configuration_object()
 

tests/support/__init__.py

@@ -39,28 +39,15 @@
 import shutil
 import stat
 import sys
+from types import SimpleNamespace
 from unittest import TestCase, main as testmain
 
+from tests.support._env import MIG_ENV
 from tests.support.configsupp import FakeConfiguration
 from tests.support.fixturesupp import _PreparedFixture
 from tests.support.suppconst import MIG_BASE, TEST_BASE, \
     TEST_DATA_DIR, TEST_OUTPUT_DIR, ENVHELP_OUTPUT_DIR
 
-from tests.support._env import MIG_ENV, PY2
-
-# Allow the use of SimpleNamespace on PY2.
-
-if PY2:
-    class SimpleNamespace(dict):
-        """Bare minimum SimpleNamespace for Python 2."""
-
-        def __getattribute__(self, name):
-            if name == '__dict__':
-                return dict(**self)
-
-            return self[name]
-else:
-    from types import SimpleNamespace
 
 
 # Provide access to a configuration file for the active environment.

tests/support/_env.py

@@ -6,6 +6,3 @@
 
 # force the chosen environment globally
 os.environ['MIG_ENV'] = MIG_ENV
-
-# expose a boolean indicating whether we are executing on Python 2
-PY2 = (sys.version_info[0] == 2)

tests/support/configsupp.py

@@ -27,9 +27,10 @@
 
 """Configuration related details within the test support library."""
 
+from types import SimpleNamespace
+
 from tests.support.loggersupp import FakeLogger
 
-from mig.shared.compat import SimpleNamespace
 from mig.shared.configuration import \
     _CONFIGURATION_ARGUMENTS, _CONFIGURATION_PROPERTIES
 

tests/support/wsgisupp.py

@@ -29,16 +29,9 @@
 from collections import namedtuple
 import codecs
 from io import BytesIO
+from urllib.parse import urlencode, urlparse
 from werkzeug.datastructures import MultiDict
 
-from tests.support._env import PY2
-
-if PY2:
-    from urllib import urlencode
-    from urlparse import urlparse
-else:
-    from urllib.parse import urlencode, urlparse
-
 
 # named type representing the tuple that is passed to WSGI handlers
 _PreparedWsgi = namedtuple('_PreparedWsgi', ['environ', 'start_response'])

tests/test_mig_shared_compat.py

@@ -33,12 +33,13 @@
 
 from tests.support import MigTestCase, testmain
 
-from mig.shared.compat import PY2, ensure_native_string
+from mig.shared.compat import ensure_native_string
 
 DUMMY_BYTECHARS = b'DEADBEEF'
 DUMMY_BYTESRAW = binascii.unhexlify('DEADBEEF') # 4 bytes
 DUMMY_UNICODE = u'UniCode123½¾µßðþđŋħĸþł@ª€£$¥©®'
 
+
 class MigSharedCompat__ensure_native_string(MigTestCase):
     """Unit test helper for the migrid code pointed to in class name"""
 
@@ -54,10 +55,7 @@ def test_raw_bytes_conversion(self):
     def test_unicode_conversion(self):
         actual = ensure_native_string(DUMMY_UNICODE)
         self.assertEqual(type(actual), str)
-        if PY2:
-            self.assertEqual(actual, DUMMY_UNICODE.encode("utf8"))
-        else:
-            self.assertEqual(actual, DUMMY_UNICODE)
+        self.assertEqual(actual, DUMMY_UNICODE)
 
 
 if __name__ == '__main__':

tests/test_mig_shared_configuration.py

@@ -31,7 +31,7 @@
 import os
 import unittest
 
-from tests.support import MigTestCase, TEST_DATA_DIR, PY2, testmain
+from tests.support import MigTestCase, TEST_DATA_DIR, testmain
 from tests.support.fixturesupp import FixtureAssertMixin
 
 from mig.shared.configuration import Configuration, \
@@ -336,7 +336,6 @@ def test_argument_include_sections_multi_ignores_other_sections(self):
 class MigSharedConfiguration__new_instance(MigTestCase, FixtureAssertMixin):
     """Coverage of programatically created Configuration instances."""
 
-    @unittest.skipIf(PY2, "Python 3 only")
     def test_default_object(self):
         prepared_fixture = self.prepareFixtureAssert(
             'mig_shared_configuration--new', fixture_format='json')

tests/test_mig_shared_functionality_cat.py

@@ -34,7 +34,7 @@
 import sys
 import unittest
 
-from tests.support import MIG_BASE, PY2, TEST_DATA_DIR, MigTestCase, testmain, \
+from tests.support import MIG_BASE, TEST_DATA_DIR, MigTestCase, testmain, \
     temppath, ensure_dirs_exist
 
 from mig.shared.base import client_id_dir
@@ -181,7 +181,6 @@ def test_file_serving_over_limit_with_storage_protocols_sftp(self):
                          "bigger than 3896 bytes - please use better "
                          "alternatives (SFTP) to retrieve large data")
 
-    @unittest.skipIf(PY2, "Python 3 only")
     def test_main_passes_environ(self):
         try:
             result = realmain(self.TEST_CLIENT_ID, {}, self.test_environ)