From da35562119feb1cf064bbc4aa47bfc69109d6a5f Mon Sep 17 00:00:00 2001 From: Peter Donis Date: Sat, 23 Nov 2019 22:31:14 -0500 Subject: [PATCH 1/9] bpo-1812: Add old-style Mac newline test to test_doctest.test_lineendings. The test_lineendings test in test_doctest did not include a test with old-style Mac line endings (\r). This commit adds one, and edits the introductory discussion in test_lineendings accordingly. --- Lib/test/test_doctest.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index f7c399e526d17f..7ee92e288655ef 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -2660,11 +2660,12 @@ def test_testfile(): r""" """ def test_lineendings(): r""" -*nix systems use \n line endings, while Windows systems use \r\n. Python +*nix systems use \n line endings, while Windows systems use \r\n, and +old Mac systems used \r, which Python still recognizes as a line ending. Python handles this using universal newline mode for reading files. Let's make sure doctest does so (issue 8473) by creating temporary test files using each -of the two line disciplines. One of the two will be the "wrong" one for the -platform the test is run on. +of the three line disciplines. At least one will not match either the universal +newline \n or os.linesep for the platform the test is run on. Windows line endings first: @@ -2687,6 +2688,16 @@ def test_lineendings(): r""" TestResults(failed=0, attempted=1) >>> os.remove(fn) +And finally old Mac line endings: + + >>> fn = tempfile.mktemp() + >>> with open(fn, 'wb') as f: + ... f.write(b'Test:\r\r >>> x = 1 + 1\r\rDone.\r') + 30 + >>> doctest.testfile(fn, module_relative=False, verbose=False) + TestResults(failed=0, attempted=1) + >>> os.remove(fn) + """ def test_testmod(): r""" From dd0284510a0c7e67cfad407593b955399f8ca990 Mon Sep 17 00:00:00 2001 From: Peter Donis Date: Sun, 24 Nov 2019 03:19:08 -0500 Subject: [PATCH 2/9] bpo-1812: Add test to test_doctest.test_lineendings that loads testfile from package with loader.get_data. The previous test_doctest.test_lineendings did not cover the case where a testfile for doctest.testfile is loaded from a package whose loader has a get_data method. This commit adds a test for that case. --- Lib/test/test_doctest.py | 56 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 7ee92e288655ef..3abfcef22d2c03 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -8,8 +8,11 @@ import os import sys import importlib +import importlib.abc +import importlib.util import unittest import tempfile +import shutil # NOTE: There are some additional tests relating to interaction with # zipimport in the test_zipimport_support test module. @@ -437,7 +440,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [] + [] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -2659,6 +2662,35 @@ def test_testfile(): r""" >>> sys.argv = save_argv """ +class TestLoader(importlib.abc.MetaPathFinder, importlib.abc.ResourceLoader): + + def find_spec(self, fullname, path, target=None): + return importlib.util.spec_from_file_location(fullname, path, loader=self) + + def get_data(self, path): + with open(path, mode='rb') as f: + return f.read() + +class TestHook: + + def __init__(self, pathdir): + self.sys_path = sys.path[:] + self.meta_path = sys.meta_path[:] + self.path_hooks = sys.path_hooks[:] + sys.path.append(pathdir) + sys.path_importer_cache.clear() + self.modules_before = sys.modules.copy() + self.importer = TestLoader() + sys.meta_path.append(self.importer) + + def remove(self): + sys.path[:] = self.sys_path + sys.meta_path[:] = self.meta_path + sys.path_hooks[:] = self.path_hooks + sys.path_importer_cache.clear() + sys.modules.clear() + sys.modules.update(self.modules_before) + def test_lineendings(): r""" *nix systems use \n line endings, while Windows systems use \r\n, and old Mac systems used \r, which Python still recognizes as a line ending. Python @@ -2698,6 +2730,28 @@ def test_lineendings(): r""" TestResults(failed=0, attempted=1) >>> os.remove(fn) +Now we test with a package loader that has a get_data method, since that +bypasses the standard universal newline handling so doctest has to do the +newline conversion itself; let's make sure it does so correctly (issue 1812). +We'll write a file inside the package that has all three kinds of line endings +in it, and use a package hook to install a custom loader; on any platform, +at least one of the line endings will raise a ValueError for inconsistent +whitespace if doctest does not correctly do the newline conversion. + + >>> dn = tempfile.mkdtemp() + >>> pkg = os.path.join(dn, "doctest_testpkg") + >>> os.mkdir(pkg) + >>> support.create_empty_file(os.path.join(pkg, "__init__.py")) + >>> fn = os.path.join(pkg, "doctest_testfile.txt") + >>> with open(fn, 'wb') as f: + ... f.write(b'Test:\r\n\r\n >>> x = 1 + 1\r\n\r\nDone.\r\nTest:\n\n >>> x = 1 + 1\n\nDone.\nTest:\r\r >>> x = 1 + 1\r\rDone.\r') + 95 + >>> hook = TestHook(dn) + >>> doctest.testfile("doctest_testfile.txt", package="doctest_testpkg", verbose=False) + TestResults(failed=0, attempted=3) + >>> hook.remove() + >>> shutil.rmtree(dn) + """ def test_testmod(): r""" From b6ad8ee30fc3d1596d06b8ec49a1c0bf00cf1208 Mon Sep 17 00:00:00 2001 From: Peter Donis Date: Sun, 24 Nov 2019 03:23:44 -0500 Subject: [PATCH 3/9] bpo-1812: Fix newline conversion when loading doctest testfile from package with loader.get_data. This commit corrects the newline conversion that is done in doctest._load_testfile when the file is being loaded from a package whose loader has a get_data method. --- Lib/doctest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/doctest.py b/Lib/doctest.py index 8fca6280b8aa6b..ba0adb36b56d69 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -211,6 +211,13 @@ def _normalize_module(module, depth=2): else: raise TypeError("Expected a module, string, or None") +def _newline_convert(data): + # We have two cases to cover and we need to make sure we do + # them in the right order + for newline in ('\r\n', '\r'): + data = data.replace(newline, '\n') + return data + def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) @@ -221,7 +228,7 @@ def _load_testfile(filename, package, module_relative, encoding): file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. - return file_contents.replace(os.linesep, '\n'), filename + return _newline_convert(file_contents), filename with open(filename, encoding=encoding) as f: return f.read(), filename From 22094c8e6a372ccf4eded8cf27009ead183278ed Mon Sep 17 00:00:00 2001 From: Peter Donis Date: Sun, 24 Nov 2019 03:26:28 -0500 Subject: [PATCH 4/9] Add Peter Donis to Misc/ACKS. --- Misc/ACKS | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/ACKS b/Misc/ACKS index 357ce024e9e8c7..341220e37e0c7b 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -411,6 +411,7 @@ Walter Dörwald Jaromir Dolecek Zsolt Dollenstein Brendan Donegan +Peter Donis Ismail Donmez Ray Donnelly Robert Donohue From 83c03d8d09edc13c4901f534b9bb9c2915c4f02d Mon Sep 17 00:00:00 2001 From: Peter Donis Date: Mon, 25 Nov 2019 21:47:10 -0500 Subject: [PATCH 5/9] Add MISC/NEWS.d blurb. --- Misc/NEWS.d/next/Tests/2019-11-25-21-46-47.bpo-1812.sAbTbY.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2019-11-25-21-46-47.bpo-1812.sAbTbY.rst diff --git a/Misc/NEWS.d/next/Tests/2019-11-25-21-46-47.bpo-1812.sAbTbY.rst b/Misc/NEWS.d/next/Tests/2019-11-25-21-46-47.bpo-1812.sAbTbY.rst new file mode 100644 index 00000000000000..7ffe90d55a4e75 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2019-11-25-21-46-47.bpo-1812.sAbTbY.rst @@ -0,0 +1,2 @@ +Fix newline handling in doctest.testfile when loading from a package whose +loader has a get_data method. Patch by Peter Donis. From efabd422baabefdabde4e10bddb9def221967f2f Mon Sep 17 00:00:00 2001 From: Peter Donis Date: Wed, 12 Feb 2020 19:51:06 -0500 Subject: [PATCH 6/9] Update Lib/test/test_doctest.py Break up long line to improve readability. Co-Authored-By: Zachary Ware --- Lib/test/test_doctest.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 3abfcef22d2c03..c10829ad300e33 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -2744,7 +2744,17 @@ def test_lineendings(): r""" >>> support.create_empty_file(os.path.join(pkg, "__init__.py")) >>> fn = os.path.join(pkg, "doctest_testfile.txt") >>> with open(fn, 'wb') as f: - ... f.write(b'Test:\r\n\r\n >>> x = 1 + 1\r\n\r\nDone.\r\nTest:\n\n >>> x = 1 + 1\n\nDone.\nTest:\r\r >>> x = 1 + 1\r\rDone.\r') + ... f.write( + ... b'Test:\r\n\r\n' + ... b' >>> x = 1 + 1\r\n\r\n' + ... b'Done.\r\n' + ... b'Test:\n\n' + ... b' >>> x = 1 + 1\n\n' + ... b'Done.\n' + ... b'Test:\r\r' + ... b' >>> x = 1+ 1\r\r' + ... b'Done.\r' + ... ) 95 >>> hook = TestHook(dn) >>> doctest.testfile("doctest_testfile.txt", package="doctest_testpkg", verbose=False) From 1d56c0561080cffd1f32f009000f9c6ac803aa0b Mon Sep 17 00:00:00 2001 From: Peter Donis Date: Wed, 12 Feb 2020 20:09:30 -0500 Subject: [PATCH 7/9] Fix typo in commit efabd422. --- Lib/test/test_doctest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index c10829ad300e33..71e3e540f74f6c 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -2752,7 +2752,7 @@ def test_lineendings(): r""" ... b' >>> x = 1 + 1\n\n' ... b'Done.\n' ... b'Test:\r\r' - ... b' >>> x = 1+ 1\r\r' + ... b' >>> x = 1 + 1\r\r' ... b'Done.\r' ... ) 95 From e1f0a2fa466138febe2f807ee63afd1cd7915e34 Mon Sep 17 00:00:00 2001 From: Peter Donis Date: Wed, 12 Feb 2020 20:11:14 -0500 Subject: [PATCH 8/9] bpo-1812: Wrap package hook test in context manager. This commit wraps the doctest.testfile test with a package loader that has a get_data method in a context manager to ensure proper removal of the extra package hook. --- Lib/test/test_doctest.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 71e3e540f74f6c..9b6d2cf4f9c5ba 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -13,6 +13,7 @@ import unittest import tempfile import shutil +import contextlib # NOTE: There are some additional tests relating to interaction with # zipimport in the test_zipimport_support test module. @@ -440,7 +441,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [] + [] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -2691,6 +2692,16 @@ def remove(self): sys.modules.clear() sys.modules.update(self.modules_before) + +@contextlib.contextmanager +def test_hook(pathdir): + hook = TestHook(pathdir) + try: + yield hook + finally: + hook.remove() + + def test_lineendings(): r""" *nix systems use \n line endings, while Windows systems use \r\n, and old Mac systems used \r, which Python still recognizes as a line ending. Python @@ -2756,10 +2767,9 @@ def test_lineendings(): r""" ... b'Done.\r' ... ) 95 - >>> hook = TestHook(dn) - >>> doctest.testfile("doctest_testfile.txt", package="doctest_testpkg", verbose=False) + >>> with test_hook(dn): + ... doctest.testfile("doctest_testfile.txt", package="doctest_testpkg", verbose=False) TestResults(failed=0, attempted=3) - >>> hook.remove() >>> shutil.rmtree(dn) """ From bb836d71b3743e8d00812f2535ad69b37080461b Mon Sep 17 00:00:00 2001 From: Peter Donis Date: Mon, 16 Mar 2020 14:22:14 -0400 Subject: [PATCH 9/9] bpo-1812: Change TestLoader to TestImporter. Change the name of the class used to test the case of a package with a loader.get_data method from TestLoader to TestImporter. --- Lib/test/test_doctest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 0c4442727dff06..9e88222e9532ff 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -2667,7 +2667,7 @@ def test_testfile(): r""" >>> sys.argv = save_argv """ -class TestLoader(importlib.abc.MetaPathFinder, importlib.abc.ResourceLoader): +class TestImporter(importlib.abc.MetaPathFinder, importlib.abc.ResourceLoader): def find_spec(self, fullname, path, target=None): return importlib.util.spec_from_file_location(fullname, path, loader=self) @@ -2685,7 +2685,7 @@ def __init__(self, pathdir): sys.path.append(pathdir) sys.path_importer_cache.clear() self.modules_before = sys.modules.copy() - self.importer = TestLoader() + self.importer = TestImporter() sys.meta_path.append(self.importer) def remove(self):