From d0a3a828be4d0e25f70b993d95cab93f7ebde7e3 Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Tue, 27 Jan 2026 17:38:08 +0100 Subject: [PATCH] Sweep the tree for everything conditional on python 2. The only thing explicitly left out is the walk relted code in fileio which is rather weedy and being tackled in a separate change. --- mig/server/grid_events.py | 10 ++--- mig/shared/base.py | 16 ++------ mig/shared/cgiscriptstub.py | 5 +-- mig/shared/compat.py | 46 +--------------------- mig/shared/configuration.py | 8 +--- mig/shared/defaults.py | 13 +----- mig/shared/output.py | 4 +- mig/shared/ssh.py | 3 +- mig/shared/url.py | 17 ++------ mig/unittest/testcore.py | 2 +- mig/wsgi-bin/migwsgi.py | 5 --- tests/support/__init__.py | 17 +------- tests/support/_env.py | 3 -- tests/support/configsupp.py | 3 +- tests/support/wsgisupp.py | 9 +---- tests/test_mig_shared_compat.py | 8 ++-- tests/test_mig_shared_configuration.py | 3 +- tests/test_mig_shared_functionality_cat.py | 3 +- tests/test_mig_shared_safeinput.py | 6 +-- tests/test_mig_wsgibin.py | 9 ++--- tests/test_support.py | 5 +-- tests/test_tests_support_wsgisupp.py | 2 +- 22 files changed, 38 insertions(+), 159 deletions(-) diff --git a/mig/server/grid_events.py b/mig/server/grid_events.py index 70c987624..1d7b28789 100755 --- a/mig/server/grid_events.py +++ b/mig/server/grid_events.py @@ -538,7 +538,7 @@ def __update_rule_monitor( # Fire 'modified' events for all dirs and files in subpath # to ensure that all rule files are loaded - for ent in scandir(src_path): + for ent in os.scandir(src_path): if ent.is_dir(follow_symlinks=True): # logger.debug('(%s) Dispatch DirCreatedEvent for: %s' @@ -1139,7 +1139,7 @@ def __update_file_monitor(self, event): # For create this occurs by eg. mkdir -p 'path/subpath/subpath2' # or 'cp -rf' - for ent in scandir(src_path): + for ent in os.scandir(src_path): if ent.is_dir(follow_symlinks=True): vgrid_sub_path = strip_base_dirs(ent.path) @@ -1491,7 +1491,7 @@ def add_vgrid_file_monitor(configuration, vgrid_name, path): # Traverse dirs for subdirs created since last run - for ent in scandir(vgrid_files_path): + for ent in os.scandir(vgrid_files_path): if ent.is_dir(follow_symlinks=True): vgrid_sub_path = strip_base_dirs(ent.path) # Force utf8 everywhere to avoid encoding issues @@ -1839,7 +1839,7 @@ def monitor(configuration, vgrid_name): configuration.vgrid_triggers) all_trigger_rules.append(rule_path) else: - for ent in scandir(vgrid_home): + for ent in os.scandir(vgrid_home): if configuration.vgrid_triggers in ent.name: rule_path = ent.path all_trigger_rules.append(rule_path) @@ -1981,7 +1981,7 @@ def monitor(configuration, vgrid_name): # Each top vgrid gets is own process - for ent in scandir(configuration.vgrid_home): + for ent in os.scandir(configuration.vgrid_home): vgrid_files_path = os.path.join(configuration.vgrid_files_home, ent.name) if os.path.isdir(ent.path) and os.path.isdir(vgrid_files_path): diff --git a/mig/shared/base.py b/mig/shared/base.py index e47807298..5910ff10b 100644 --- a/mig/shared/base.py +++ b/mig/shared/base.py @@ -736,23 +736,13 @@ def native_args(argv): https://docs.python.org/3/library/sys.html#sys.argv for further details. """ - if sys.version_info[0] >= 3: - return [os.fsencode(arg) for arg in argv] - else: - return argv + return [os.fsencode(arg) for arg in argv] def NativeStringIO(initial_value=''): - """Mock StringIO pseudo-class to create a StringIO matching the native - string coding form. That is a BytesIO with utf8 on python 2 and unicode - StringIO otherwise. Optional string helpers are automatically converted - accordingly. + """Pseudo-class wrapper to return a StringIO. This is a unicode StringIO. """ - if sys.version_info[0] >= 3: - return io.StringIO(initial_value) - else: - from StringIO import StringIO - return StringIO(initial_value) + return io.StringIO(initial_value) def DefaultStringIO(initial_value=''): diff --git a/mig/shared/cgiscriptstub.py b/mig/shared/cgiscriptstub.py index 84c231ed1..850c371c2 100755 --- a/mig/shared/cgiscriptstub.py +++ b/mig/shared/cgiscriptstub.py @@ -117,9 +117,8 @@ def finish_cgi_script(configuration, backend, output_format, ret_code, ret_msg, #logger.debug("flush stdout") sys.stdout.flush() #logger.debug("write content: %s" % [output[:64], '..', output[-64:]]) - # NOTE: always output native strings to stdout but use raw buffer - # for byte output on py3 as explained above. - if sys.version_info[0] < 3 or is_default_str_coding(output): + # NOTE: use raw buffer for byte output as explained above + if is_default_str_coding(output): sys.stdout.write(output) else: sys.stdout.buffer.write(output) diff --git a/mig/shared/compat.py b/mig/shared/compat.py index 01069ce43..64f05cd46 100644 --- a/mig/shared/compat.py +++ b/mig/shared/compat.py @@ -36,29 +36,10 @@ import codecs import io import sys -# NOTE: StringIO is only available in python2 -try: - import StringIO -except ImportError: - StringIO = None -PY2 = sys.version_info[0] < 3 _TYPE_UNICODE = type(u"") -if PY2: - class SimpleNamespace(dict): - """Bare minimum SimpleNamespace for Python 2.""" - - def __getattribute__(self, name): - if name == '__dict__': - return dict(**self) - - return self[name] -else: - from types import SimpleNamespace - - def _is_unicode(val): """Return boolean indicating if the value is a unicode string. @@ -72,34 +53,9 @@ def ensure_native_string(string_or_bytes): """Given a supplied input which can be either a string or bytes return a representation providing string operations while ensuring that its contents represent a valid series of textual characters. - - Arrange identical operation across python 2 and 3 - specifically, - the presence of invalid UTF-8 bytes (thus the input not being a - valid textual string) will trigger a UnicodeDecodeError on PY3. - Force the same to occur on PY2. """ - if PY2: - # Simulate decoding done by PY3 to trigger identical exceptions - # note the use of a forced "utf8" encoding value: this function - # is generally used to wrap, for example, substitutions of values - # into strings that are defined in the source code. In Python 3 - # these are mandated to be UTF-8, and thus decoding as "utf8" is - # what will be attempted on supplied input. Match it. - textual_output = codecs.encode(string_or_bytes, 'utf8') - elif not _is_unicode(string_or_bytes): + if not _is_unicode(string_or_bytes): textual_output = str(string_or_bytes, 'utf8') else: textual_output = string_or_bytes return textual_output - - -def NativeStringIO(initial_value=''): - """Mock StringIO pseudo-class to create a StringIO matching the native - string coding form. That is a BytesIO with utf8 on python 2 and unicode - StringIO otherwise. Optional string helpers are automatically converted - accordingly. - """ - if PY2 and StringIO is not None: - return StringIO.StringIO(initial_value) - else: - return io.StringIO(initial_value) diff --git a/mig/shared/configuration.py b/mig/shared/configuration.py index 09a869905..111d05e79 100644 --- a/mig/shared/configuration.py +++ b/mig/shared/configuration.py @@ -34,6 +34,7 @@ from builtins import object import base64 +from configparser import ConfigParser import copy import datetime import functools @@ -50,13 +51,6 @@ standard_library.install_aliases() -# NOTE: should be handled by future aliases but fails in PAM C extension -if sys.version_info[0] < 3: - # NOTE: always use native py2 version here - new one causes unicode mess - from ConfigParser import ConfigParser -else: - from configparser import ConfigParser - # NOTE: protect migrid import from autopep8 reordering try: from mig.shared.base import force_native_str diff --git a/mig/shared/defaults.py b/mig/shared/defaults.py index c05c886c8..c71cfa541 100644 --- a/mig/shared/defaults.py +++ b/mig/shared/defaults.py @@ -37,17 +37,8 @@ MIG_BASE = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) MIG_ENV = os.getenv('MIG_ENV', 'default') -# NOTE: python3 switched strings to use unicode by default in contrast to bytes -# in python2. File systems remain with utf8 however so we need to -# carefully handle a lot of cases of either encoding to utf8 or decoding -# to unicode depending on the python used. -# Please refer to the helpers in shared.base for actual handling of it. -if sys.version_info[0] >= 3: - default_str_coding = 'unicode' - default_fs_coding = 'utf8' -else: - default_str_coding = 'utf8' - default_fs_coding = 'utf8' +default_str_coding = 'unicode' +default_fs_coding = 'utf8' CODING_KINDS = (STR_KIND, FS_KIND) = ('__STR__', '__FS__') diff --git a/mig/shared/output.py b/mig/shared/output.py index 1d9bae046..968228fe4 100644 --- a/mig/shared/output.py +++ b/mig/shared/output.py @@ -651,7 +651,7 @@ def txt_format(configuration, ret_val, ret_msg, out_obj): lines = [status_line] + lines # NOTE: careful handling required for binary on python3+ - if sys.version_info[0] > 2 and binary_output: + if binary_output: return b''.join(lines) else: return ''.join(lines) @@ -2646,7 +2646,7 @@ def html_format(configuration, ret_val, ret_msg, out_obj): user_widgets)) # NOTE: careful handling required for binary on python3+ - if sys.version_info[0] > 2 and binary_output: + if binary_output: return b''.join(lines) else: return '\n'.join(lines) diff --git a/mig/shared/ssh.py b/mig/shared/ssh.py index 6df8d12a9..e5f292cbd 100644 --- a/mig/shared/ssh.py +++ b/mig/shared/ssh.py @@ -44,9 +44,8 @@ # Paramiko not available - imported fom griddaemons so fail gracefully paramiko = None -from mig.shared.base import client_id_dir, force_utf8 +from mig.shared.base import client_id_dir, force_utf8, NativeStringIO from mig.shared.conf import get_resource_exe, get_configuration_object -from mig.shared.compat import NativeStringIO from mig.shared.defaults import ssh_conf_dir from mig.shared.safeeval import subprocess_popen, subprocess_pipe diff --git a/mig/shared/url.py b/mig/shared/url.py index f50e07a02..121d7680d 100644 --- a/mig/shared/url.py +++ b/mig/shared/url.py @@ -42,20 +42,9 @@ import base64 import os import sys - -# NOTE: moved to urllib.parse in python3 and are re-exposed with future. -# Other modules should import helpers from here for consistency. -# TODO: handle the unicode returned by python3 and future versions! -# Perhaps switch to suggested "easiest option" from -# http://python-future.org/compatible_idioms.html#urllib-module -# once we have unicode/bytecode mix-up sorted out. -if sys.version_info[0] >= 3: - from urllib.parse import quote, unquote, urlencode, parse_qs, parse_qsl, \ - urlsplit, urlparse, urljoin - from urllib.request import urlopen -else: - from urllib import quote, unquote, urlencode, urlopen - from urlparse import parse_qs, parse_qsl, urlsplit, urlparse, urljoin +from urllib.parse import quote, unquote, urlencode, parse_qs, parse_qsl, \ + urlsplit, urlparse, urljoin +from urllib.request import urlopen try: from mig.shared.base import force_utf8, force_native_str diff --git a/mig/unittest/testcore.py b/mig/unittest/testcore.py index a48333380..3eea925a0 100644 --- a/mig/unittest/testcore.py +++ b/mig/unittest/testcore.py @@ -38,7 +38,7 @@ sys.path.append(os.path.realpath( os.path.join(os.path.dirname(__file__), "../.."))) -from tests.support import MIG_BASE, PY2, is_path_within +from tests.support import MIG_BASE, is_path_within from mig.shared.base import client_id_dir, client_dir_id, get_short_id, \ invisible_path, allow_script, brief_list diff --git a/mig/wsgi-bin/migwsgi.py b/mig/wsgi-bin/migwsgi.py index 073958880..9d3e6d98d 100755 --- a/mig/wsgi-bin/migwsgi.py +++ b/mig/wsgi-bin/migwsgi.py @@ -249,11 +249,6 @@ def application(environ, start_response, configuration=None, # (sys.version_info, ), file=environ['wsgi.errors']) # print("DEBUG: path %s" % sys.path, file=environ['wsgi.errors']) - # NOTE: redirect stdout to stderr in python 2 only. It breaks logger in 3 - # and stdout redirection apparently is already handled there. - if sys.version_info[0] < 3: - sys.stdout = sys.stderr - if configuration is None: configuration = get_configuration_object() diff --git a/tests/support/__init__.py b/tests/support/__init__.py index b3c60ae62..dca2c8923 100644 --- a/tests/support/__init__.py +++ b/tests/support/__init__.py @@ -39,28 +39,15 @@ import shutil import stat import sys +from types import SimpleNamespace from unittest import TestCase, main as testmain +from tests.support._env import MIG_ENV from tests.support.configsupp import FakeConfiguration from tests.support.fixturesupp import _PreparedFixture from tests.support.suppconst import MIG_BASE, TEST_BASE, \ TEST_DATA_DIR, TEST_OUTPUT_DIR, ENVHELP_OUTPUT_DIR -from tests.support._env import MIG_ENV, PY2 - -# Allow the use of SimpleNamespace on PY2. - -if PY2: - class SimpleNamespace(dict): - """Bare minimum SimpleNamespace for Python 2.""" - - def __getattribute__(self, name): - if name == '__dict__': - return dict(**self) - - return self[name] -else: - from types import SimpleNamespace # Provide access to a configuration file for the active environment. diff --git a/tests/support/_env.py b/tests/support/_env.py index 2c71386a4..6a80f43ce 100644 --- a/tests/support/_env.py +++ b/tests/support/_env.py @@ -6,6 +6,3 @@ # force the chosen environment globally os.environ['MIG_ENV'] = MIG_ENV - -# expose a boolean indicating whether we are executing on Python 2 -PY2 = (sys.version_info[0] == 2) diff --git a/tests/support/configsupp.py b/tests/support/configsupp.py index 0846e465d..02c73f2c9 100644 --- a/tests/support/configsupp.py +++ b/tests/support/configsupp.py @@ -27,9 +27,10 @@ """Configuration related details within the test support library.""" +from types import SimpleNamespace + from tests.support.loggersupp import FakeLogger -from mig.shared.compat import SimpleNamespace from mig.shared.configuration import \ _CONFIGURATION_ARGUMENTS, _CONFIGURATION_PROPERTIES diff --git a/tests/support/wsgisupp.py b/tests/support/wsgisupp.py index 2451967d1..4de99822a 100644 --- a/tests/support/wsgisupp.py +++ b/tests/support/wsgisupp.py @@ -29,16 +29,9 @@ from collections import namedtuple import codecs from io import BytesIO +from urllib.parse import urlencode, urlparse from werkzeug.datastructures import MultiDict -from tests.support._env import PY2 - -if PY2: - from urllib import urlencode - from urlparse import urlparse -else: - from urllib.parse import urlencode, urlparse - # named type representing the tuple that is passed to WSGI handlers _PreparedWsgi = namedtuple('_PreparedWsgi', ['environ', 'start_response']) diff --git a/tests/test_mig_shared_compat.py b/tests/test_mig_shared_compat.py index 05f2de2bc..1d91b75d3 100644 --- a/tests/test_mig_shared_compat.py +++ b/tests/test_mig_shared_compat.py @@ -33,12 +33,13 @@ from tests.support import MigTestCase, testmain -from mig.shared.compat import PY2, ensure_native_string +from mig.shared.compat import ensure_native_string DUMMY_BYTECHARS = b'DEADBEEF' DUMMY_BYTESRAW = binascii.unhexlify('DEADBEEF') # 4 bytes DUMMY_UNICODE = u'UniCode123½¾µßðþđŋħĸþł@ª€£$¥©®' + class MigSharedCompat__ensure_native_string(MigTestCase): """Unit test helper for the migrid code pointed to in class name""" @@ -54,10 +55,7 @@ def test_raw_bytes_conversion(self): def test_unicode_conversion(self): actual = ensure_native_string(DUMMY_UNICODE) self.assertEqual(type(actual), str) - if PY2: - self.assertEqual(actual, DUMMY_UNICODE.encode("utf8")) - else: - self.assertEqual(actual, DUMMY_UNICODE) + self.assertEqual(actual, DUMMY_UNICODE) if __name__ == '__main__': diff --git a/tests/test_mig_shared_configuration.py b/tests/test_mig_shared_configuration.py index 1108ea3e0..cde342195 100644 --- a/tests/test_mig_shared_configuration.py +++ b/tests/test_mig_shared_configuration.py @@ -31,7 +31,7 @@ import os import unittest -from tests.support import MigTestCase, TEST_DATA_DIR, PY2, testmain +from tests.support import MigTestCase, TEST_DATA_DIR, testmain from tests.support.fixturesupp import FixtureAssertMixin from mig.shared.configuration import Configuration, \ @@ -336,7 +336,6 @@ def test_argument_include_sections_multi_ignores_other_sections(self): class MigSharedConfiguration__new_instance(MigTestCase, FixtureAssertMixin): """Coverage of programatically created Configuration instances.""" - @unittest.skipIf(PY2, "Python 3 only") def test_default_object(self): prepared_fixture = self.prepareFixtureAssert( 'mig_shared_configuration--new', fixture_format='json') diff --git a/tests/test_mig_shared_functionality_cat.py b/tests/test_mig_shared_functionality_cat.py index 619c3dd4e..ee3db8c96 100644 --- a/tests/test_mig_shared_functionality_cat.py +++ b/tests/test_mig_shared_functionality_cat.py @@ -34,7 +34,7 @@ import sys import unittest -from tests.support import MIG_BASE, PY2, TEST_DATA_DIR, MigTestCase, testmain, \ +from tests.support import MIG_BASE, TEST_DATA_DIR, MigTestCase, testmain, \ temppath, ensure_dirs_exist from mig.shared.base import client_id_dir @@ -181,7 +181,6 @@ def test_file_serving_over_limit_with_storage_protocols_sftp(self): "bigger than 3896 bytes - please use better " "alternatives (SFTP) to retrieve large data") - @unittest.skipIf(PY2, "Python 3 only") def test_main_passes_environ(self): try: result = realmain(self.TEST_CLIENT_ID, {}, self.test_environ) diff --git a/tests/test_mig_shared_safeinput.py b/tests/test_mig_shared_safeinput.py index 4a01dddd8..ede5204f3 100644 --- a/tests/test_mig_shared_safeinput.py +++ b/tests/test_mig_shared_safeinput.py @@ -39,14 +39,10 @@ valid_printable, valid_base_url, valid_url, valid_complex_url, \ VALID_NAME_CHARACTERS -PY2 = sys.version_info[0] == 2 - def as_string_of_unicode(value): assert isinstance(value, basestring) - if not is_string_of_unicode(value): - assert PY2, "unreachable unless Python 2" - return unicode(codecs.decode(value, 'utf8')) + assert is_string_of_unicode(value) return value diff --git a/tests/test_mig_wsgibin.py b/tests/test_mig_wsgibin.py index 857d03b99..3a633e320 100644 --- a/tests/test_mig_wsgibin.py +++ b/tests/test_mig_wsgibin.py @@ -29,24 +29,21 @@ import codecs from configparser import ConfigParser +from html.parser import HTMLParser import importlib import os import stat import sys +from types import SimpleNamespace -from tests.support import PY2, MIG_BASE, MigTestCase, testmain, is_path_within +from tests.support import MIG_BASE, MigTestCase, testmain, is_path_within from tests.support.snapshotsupp import SnapshotAssertMixin from tests.support.wsgisupp import prepare_wsgi, WsgiAssertMixin from mig.shared.base import client_id_dir, client_dir_id, get_short_id, \ invisible_path, allow_script, brief_list -from mig.shared.compat import SimpleNamespace import mig.shared.returnvalues as returnvalues -if PY2: - from HTMLParser import HTMLParser -else: - from html.parser import HTMLParser class DocumentBasicsHtmlParser(HTMLParser): diff --git a/tests/test_support.py b/tests/test_support.py index 56b74f594..73f3834e4 100644 --- a/tests/test_support.py +++ b/tests/test_support.py @@ -32,7 +32,7 @@ import sys import unittest -from tests.support import MigTestCase, PY2, testmain, temppath, \ +from tests.support import MigTestCase, testmain, temppath, \ AssertOver, FakeConfiguration from mig.shared.conf import get_configuration_object @@ -83,7 +83,6 @@ def test_requires_requesting_a_configuration(self): self.assertEqual(str(theexception), "configuration access but testcase did not request it") - @unittest.skipIf(PY2, "Python 3 only") def test_unclosed_files_are_recorded(self): tmp_path = temppath("support-unclosed", self) @@ -160,7 +159,7 @@ def _provide_configuration(self): return 'testconfig' def test_provides_the_test_configuration(self): - expected_last_dir = 'testconfs-py2' if PY2 else 'testconfs-py3' + expected_last_dir = 'testconfs-py3' configuration = self.configuration diff --git a/tests/test_tests_support_wsgisupp.py b/tests/test_tests_support_wsgisupp.py index 4adcb975d..ca452d91a 100644 --- a/tests/test_tests_support_wsgisupp.py +++ b/tests/test_tests_support_wsgisupp.py @@ -28,7 +28,7 @@ """Unit tests for the tests module pointed to in the filename""" import unittest -from mig.shared.compat import SimpleNamespace +from types import SimpleNamespace from tests.support import AssertOver from tests.support.wsgisupp import prepare_wsgi