Merge branch 'main' into 3.6

Author: slozier

eng/scripts/generate.py

@@ -120,10 +120,13 @@ def finally_block(self):
         self.writeline("} finally {")
         self.indent()
 
-    def exit_block(self, text=None, **kw):
+    def exit_block(self, text=None):
         self.dedent()
         if text:
-            self.writeline("} " + text, **kw)
+            if text == ";":
+                 self.writeline("}" + text)
+            else:
+                self.writeline("} " + text)
         else:
             self.writeline('}')
 

eng/scripts/generate_encoding_aliases.py

@@ -3,24 +3,44 @@
 # See the LICENSE file in the project root for more information.
 
 import encodings
+import itertools
+import sys
 
 from generate import generate
 
 def gen_aliases(cw):
     cw.writeline("// Based on encodings.aliases")
     cw.enter_block("var d = new Dictionary<string, string>")
 
-    for codec in sorted(set(encodings.aliases.aliases.values())):
+    aliases = encodings.aliases.aliases
+    aliases_36 = {
+        "kz_1048": "kz1048",
+        "rk1048": "kz1048",
+        "strk1048_2002": "kz1048",
+    }
+    codecs_36 = {v: "PYTHON_36_OR_GREATER" for v in aliases_36.values()}
+    if sys.version_info >= (3, 6):
+        for k, v in aliases_36.items():
+            assert aliases[k] == v
+    aliases = {**aliases, **aliases_36}
+
+    sorted_aliases = sorted(aliases.items(), key=lambda kv: (kv[1], kv[0]))
+    for codec, group in itertools.groupby(sorted_aliases, key=lambda kv: kv[1]):
+        condition = codecs_36.get(codec)
+
         e = encodings.search_function(codec)
-        if e is None or not e._is_text_encoding or e.name == "mbcs":
+        if not condition and (e is None or not e._is_text_encoding or e.name == "mbcs"):
             continue
 
         cw.writeline()
-        aliases = sorted(alias for alias, aliased_codec in encodings.aliases.aliases.items() if aliased_codec == codec)
-        for alias in aliases:
+        if condition:
+            cw.writeline("#if {0}".format(condition))
+        for alias, codec in group:
             qalias = '"{0}"'.format(alias)
             qcodec = '"{0}"'.format(codec)
             cw.writeline('{{ {0:24} , {1:24} }},'.format(qalias, qcodec))
+        if condition:
+            cw.writeline("#endif")
 
     cw.exit_block(";")
 

src/core/IronPython/IronPython.csproj

@@ -63,7 +63,7 @@
     <EmbeddedResource Include="Modules\unicodedata\IPyUnicodeData.txt.gz" />
     <EmbeddedResource Include="Modules\unicodedata\IPyUnicodeData-3.2.0.txt.gz" />
     <EmbeddedResource Include="..\IronPython.StdLib\lib\importlib\_bootstrap.py" Link="Modules\_bootstrap.py" />
-    <EmbeddedResource Include="..\IronPython.StdLib\lib\importlib\_bootstrap_external.py" Link="Modules\_bootstrap_external.py" />
+    <EmbeddedResource Condition="'$(MajorVersion)$(MinorVersion)' >= '36'" Include="..\IronPython.StdLib\lib\importlib\_bootstrap_external.py" Link="Modules\_bootstrap_external.py" />
   </ItemGroup>
 
   <ItemGroup>

src/core/IronPython/Runtime/Operations/StringOps.Generated.cs

@@ -253,9 +253,11 @@ static partial class CodecsInfo {
 
                     { "cskoi8r"                , "koi8_r"                 },
 
+                    #if PYTHON_36_OR_GREATER
                     { "kz_1048"                , "kz1048"                 },
                     { "rk1048"                 , "kz1048"                 },
                     { "strk1048_2002"          , "kz1048"                 },
+                    #endif
 
                     { "8859"                   , "latin_1"                },
                     { "cp819"                  , "latin_1"                },
@@ -320,7 +322,7 @@ static partial class CodecsInfo {
                     { "utf8"                   , "utf_8"                  },
                     { "utf8_ucs2"              , "utf_8"                  },
                     { "utf8_ucs4"              , "utf_8"                  },
-                } ;
+                };
 
                 // *** END GENERATED CODE ***
 

tests/IronPython.Tests/Cases/CPythonCasesManifest.ini

@@ -531,9 +531,8 @@ Reason=Only valid for Unix
 [CPython.test_gzip]
 IsolationLevel=PROCESS # https://github.com/IronLanguages/ironpython3/issues/1440
 
-# TODO: 3.6
-#[CPython.test_heapq]
-#Ignore=true
+[CPython.test_heapq] # IronPython.test_heapq_stdlib
+Ignore=true
 
 [CPython.test_httplib] # IronPython.test_httplib_stdlib
 Ignore=true
@@ -833,10 +832,8 @@ RunCondition=$(IS_POSIX)
 Ignore=true
 Reason=unittest.case.SkipTest: No module named 'readline'
 
-# TODO: 3.6
-[CPython.test_regrtest]
-IsolationLevel=PROCESS
-#Ignore=true # lots of failures
+[CPython.test_regrtest] # IronPython.test_regrtest_stdlib
+Ignore=true
 
 [CPython.test_reprlib]
 Ignore=true

tests/IronPython.Tests/Cases/IronPythonCasesManifest.ini

@@ -105,6 +105,9 @@ RetryCount=2
 IsolationLevel=PROCESS
 Arguments=-X:CompilationThreshold 0 "$(TEST_FILE)" # ensure CompilationThreshold is 0 otherwise tests are meaningless!
 
+[IronPython.test_regrtest_stdlib]
+IsolationLevel=PROCESS
+
 [IronPython.test_set_stdlib]
 RunCondition=NOT $(IS_MONO) # weakref failures; https://github.com/IronLanguages/ironpython3/issues/544
 IsolationLevel=PROCESS # Also weakref failures; https://github.com/IronLanguages/ironpython3/issues/489

tests/suite/test_datetime_stdlib.py

@@ -19,7 +19,6 @@ def load_tests(loader, standard_tests, pattern):
         failing_tests = [
             test.datetimetester.TestDate('test_backdoor_resistance'),
             test.datetimetester.TestDate('test_insane_fromtimestamp'),
-            test.datetimetester.TestDateTime('test_backdoor_resistance'),
             test.datetimetester.TestDateTime('test_extreme_timedelta'),
             test.datetimetester.TestDateTime('test_insane_fromtimestamp'),
             test.datetimetester.TestDateTime('test_insane_utcfromtimestamp'),
@@ -46,6 +45,10 @@ def load_tests(loader, standard_tests, pattern):
             test.datetimetester.TestTimeTZ('test_zones'),
             test.datetimetester.TestTimeZone('test_constructor'),
         ]
+        if sys.version_info < (3, 6):
+            failing_tests += [
+                test.datetimetester.TestDateTime('test_backdoor_resistance'),
+            ]
         if sys.version_info >= (3, 6):
             failing_tests += [
                 test.datetimetester.IranTest('test_folds'), # https://github.com/IronLanguages/ironpython3/issues/1459
@@ -60,10 +63,12 @@ def load_tests(loader, standard_tests, pattern):
                 test.datetimetester.TestDateTime('test_compat_unpickle'), # TypeError: function takes at least 3 arguments (1 given)
                 test.datetimetester.TestDateTime('test_isoformat'), # TypeError: isoformat() got an unexpected keyword argument 'timespec'
                 test.datetimetester.TestDateTime('test_subclass_replace'), # TypeError: replace() got an unexpected keyword argument 'year'
+                test.datetimetester.TestDateTime('test_timestamp_limits'), # ValueError: The added or subtracted value results in an un-representable DateTime.
                 test.datetimetester.TestDateTimeTZ('test_bad_constructor_arguments'), # https://github.com/IronLanguages/ironpython3/issues/1459
                 test.datetimetester.TestDateTimeTZ('test_compat_unpickle'), # TypeError: function takes at least 3 arguments (2 given)
                 test.datetimetester.TestDateTimeTZ('test_isoformat'), # TypeError: isoformat() got an unexpected keyword argument 'timespec'
                 test.datetimetester.TestDateTimeTZ('test_subclass_replace'), # TypeError: replace() got an unexpected keyword argument 'year'
+                test.datetimetester.TestDateTimeTZ('test_timestamp_limits'), # ValueError: The added or subtracted value results in an un-representable DateTime.
                 test.datetimetester.TestLocalTimeDisambiguation('test_comparison'), # https://github.com/IronLanguages/ironpython3/issues/1459
                 test.datetimetester.TestLocalTimeDisambiguation('test_constructors'), # https://github.com/IronLanguages/ironpython3/issues/1459
                 test.datetimetester.TestLocalTimeDisambiguation('test_dst'), # https://github.com/IronLanguages/ironpython3/issues/1459
@@ -82,35 +87,11 @@ def load_tests(loader, standard_tests, pattern):
                 test.datetimetester.TestLocalTimeDisambiguation('test_vilnius_1941_toutc'), # AssertionError: '06/23/41 19:59:59 ' != 'Mon Jun 23 19:59:59 1941 UTC'
                 test.datetimetester.TestModule('test_divide_and_round'), # AttributeError: 'module' object has no attribute '_divide_and_round'
                 test.datetimetester.TestSubclassDateTime('test_bad_constructor_arguments'), # https://github.com/IronLanguages/ironpython3/issues/1459
+                test.datetimetester.TestSubclassDateTime('test_combine'), # TypeError: combine() takes exactly 2 arguments (3 given)
+                test.datetimetester.TestSubclassDateTime('test_compat_unpickle'), # TypeError: function takes at least 3 arguments (1 given)
                 test.datetimetester.TestSubclassDateTime('test_isoformat'), # TypeError: isoformat() got an unexpected keyword argument 'timespec'
                 test.datetimetester.TestSubclassDateTime('test_subclass_replace'), # TypeError: replace() got an unexpected keyword argument 'year'
-                test.datetimetester.TestTime('test_backdoor_resistance'), # AssertionError: "^bad tzinfo state arg$" does not match "expected Int32, got bytes"
-                test.datetimetester.TestTime('test_compat_unpickle'), # TypeError: expected Int32, got str
-                test.datetimetester.TestTime('test_isoformat'), # TypeError: isoformat() takes no arguments (1 given)
-                test.datetimetester.TestTime('test_subclass_replace'), # AssertionError: <class '_datetime.time'> is not <class 'test.datetimetester.TimeSubclass'>
-                test.datetimetester.TestTimeDelta('test_issue31293'), # ZeroDivisionError: Attempted to divide by zero.
-                test.datetimetester.TestTimeTZ('test_backdoor_resistance'), # AssertionError: "^bad tzinfo state arg$" does not match "expected Int32, got bytes"
-                test.datetimetester.TestTimeTZ('test_compat_unpickle'), # TypeError: expected Int32, got str
-                test.datetimetester.TestTimeTZ('test_isoformat'), # TypeError: isoformat() takes no arguments (1 given)
-                test.datetimetester.TestTimeTZ('test_subclass_replace'), # AssertionError: <class '_datetime.time'> is not <class 'test.datetimetester.TimeSubclass'>
-                test.datetimetester.ZoneInfoTest('test_folds'), # https://github.com/IronLanguages/ironpython3/issues/1459
-                test.datetimetester.ZoneInfoTest('test_gaps'), # https://github.com/IronLanguages/ironpython3/issues/1459
-                test.datetimetester.ZoneInfoTest('test_system_transitions'), # AttributeError: 'module' object has no attribute 'tzset'
-                test.datetimetester.TestSubclassDateTime('test_subclass_replace'), # TypeError: replace() got an unexpected keyword argument 'year'
-                test.datetimetester.TestTime('test_backdoor_resistance'), # AssertionError: "^bad tzinfo state arg$" does not match "expected Int32, got bytes"
-                test.datetimetester.TestTime('test_compat_unpickle'), # TypeError: expected Int32, got str
-                test.datetimetester.TestTime('test_isoformat'), # TypeError: isoformat() takes no arguments (1 given)
-                test.datetimetester.TestTime('test_subclass_replace'), # AssertionError: <class '_datetime.time'> is not <class 'test.datetimetester.TimeSubclass'>
-                test.datetimetester.TestTimeDelta('test_computations'), # rounding differences
-                test.datetimetester.TestTimeDelta('test_issue31293'), # ZeroDivisionError: Attempted to divide by zero.
-                test.datetimetester.TestTimeTZ('test_backdoor_resistance'), # AssertionError: "^bad tzinfo state arg$" does not match "expected Int32, got bytes"
-                test.datetimetester.TestTimeTZ('test_compat_unpickle'), # TypeError: expected Int32, got str
-                test.datetimetester.TestTimeTZ('test_isoformat'), # TypeError: isoformat() takes no arguments (1 given)
-                test.datetimetester.TestTimeTZ('test_subclass_replace'), # AssertionError: <class '_datetime.time'> is not <class 'test.datetimetester.TimeSubclass'>
-                test.datetimetester.ZoneInfoTest('test_folds'), # https://github.com/IronLanguages/ironpython3/issues/1459
-                test.datetimetester.ZoneInfoTest('test_gaps'), # https://github.com/IronLanguages/ironpython3/issues/1459
-                test.datetimetester.ZoneInfoTest('test_system_transitions'), # AttributeError: 'module' object has no attribute 'tzset'
-                test.datetimetester.TestSubclassDateTime('test_subclass_replace'), # TypeError: replace() got an unexpected keyword argument 'year'
+                test.datetimetester.TestSubclassDateTime('test_timestamp_limits'), # ValueError: The added or subtracted value results in an un-representable DateTime.
                 test.datetimetester.TestTime('test_backdoor_resistance'), # AssertionError: "^bad tzinfo state arg$" does not match "expected Int32, got bytes"
                 test.datetimetester.TestTime('test_compat_unpickle'), # TypeError: expected Int32, got str
                 test.datetimetester.TestTime('test_isoformat'), # TypeError: isoformat() takes no arguments (1 given)

tests/suite/test_heapq_stdlib.py

@@ -0,0 +1,35 @@
+# Licensed to the .NET Foundation under one or more agreements.
+# The .NET Foundation licenses this file to you under the Apache 2.0 License.
+# See the LICENSE file in the project root for more information.
+
+##
+## Run selected tests from test_heapq from StdLib
+##
+
+import sys
+
+from iptest import is_ironpython, generate_suite, run_test
+
+import test.test_heapq
+
+def load_tests(loader, standard_tests, pattern):
+    tests = loader.loadTestsFromModule(test.test_heapq, pattern=pattern)
+
+    if is_ironpython:
+        failing_tests = []
+        if sys.version_info >= (3, 6):
+            failing_tests += [
+                test.test_heapq.TestErrorHandlingC('test_comparison_operator_modifiying_heap'), # TypeError: '<' not supported between instances of 'EvilClass' and 'int'
+                test.test_heapq.TestErrorHandlingC('test_comparison_operator_modifiying_heap_two_heaps'), # AssertionError: (<class 'IndexError'>, <class 'RuntimeError'>) not raised by heappush
+                test.test_heapq.TestErrorHandlingPython('test_comparison_operator_modifiying_heap'), # TypeError: '<' not supported between instances of 'EvilClass' and 'int'
+                test.test_heapq.TestModules('test_c_functions'), # AssertionError: 'heapq' != '_heapq'
+            ]
+
+        skip_tests = []
+
+        return generate_suite(tests, failing_tests, skip_tests)
+
+    else:
+        return tests
+
+run_test(__name__)

tests/suite/test_io_stdlib.py

@@ -83,7 +83,6 @@ def load_tests(loader, standard_tests, pattern):
             test.test_io.CMiscIOTest('test_readinto_buffer_overflow'), # IndexError: Index was outside the bounds of the array.
             test.test_io.CMiscIOTest('test_warn_on_dealloc'), # AssertionError: ResourceWarning not triggered
             test.test_io.CMiscIOTest('test_warn_on_dealloc_fd'), # AssertionError: ResourceWarning not triggered
-            test.test_io.PyMiscIOTest('test_io_after_close'), # AttributeError: 'FileIO' object has no attribute 'read1'
             test.test_io.PyMiscIOTest('test_nonblock_pipe_write_bigbuf'), # AttributeError: 'module' object has no attribute 'fcntl'
             test.test_io.PyMiscIOTest('test_nonblock_pipe_write_smallbuf'), # AttributeError: 'module' object has no attribute 'fcntl'
             test.test_io.PyMiscIOTest('test_warn_on_dealloc'), # AssertionError: ResourceWarning not triggered
@@ -102,6 +101,7 @@ def load_tests(loader, standard_tests, pattern):
                 test.test_io.CBufferedWriterTest('test_garbage_collection'), # AssertionError: filter ('', ResourceWarning) did not catch any warning
                 test.test_io.CBufferedRandomTest('test_garbage_collection'), # AssertionError: filter ('', ResourceWarning) did not catch any warning
                 test.test_io.CTextIOWrapperTest('test_garbage_collection'), # AssertionError: filter ('', ResourceWarning) did not catch any warning
+                test.test_io.PyMiscIOTest('test_io_after_close'), # AttributeError: 'FileIO' object has no attribute 'read1'
                 test.test_io.PyMiscIOTest('test_pickling'), # AssertionError: TypeError not raised by _dumps
             ]
         if sys.version_info >= (3, 6):

tests/suite/test_regrtest_stdlib.py

@@ -0,0 +1,62 @@
+# Licensed to the .NET Foundation under one or more agreements.
+# The .NET Foundation licenses this file to you under the Apache 2.0 License.
+# See the LICENSE file in the project root for more information.
+
+##
+## Run selected tests from test_regrtest from StdLib
+##
+
+import sys
+
+from iptest import is_ironpython, generate_suite, run_test
+
+import test.test_regrtest
+
+def load_tests(loader, standard_tests, pattern):
+    tests = loader.loadTestsFromModule(test.test_regrtest, pattern=pattern)
+
+    if is_ironpython:
+        failing_tests = []
+
+        skip_tests = []
+        if sys.version_info >= (3, 6):
+            skip_tests += [
+                # TODO: these all fail (and are slow)
+                test.test_regrtest.ArgsTestCase('test_coverage'),
+                test.test_regrtest.ArgsTestCase('test_crashed'),
+                test.test_regrtest.ArgsTestCase('test_env_changed'),
+                test.test_regrtest.ArgsTestCase('test_failing_test'),
+                test.test_regrtest.ArgsTestCase('test_forever'),
+                test.test_regrtest.ArgsTestCase('test_fromfile'),
+                test.test_regrtest.ArgsTestCase('test_huntrleaks'),
+                test.test_regrtest.ArgsTestCase('test_huntrleaks_fd_leak'),
+                test.test_regrtest.ArgsTestCase('test_interrupted'),
+                test.test_regrtest.ArgsTestCase('test_list_cases'),
+                test.test_regrtest.ArgsTestCase('test_list_tests'),
+                test.test_regrtest.ArgsTestCase('test_matchfile'),
+                test.test_regrtest.ArgsTestCase('test_no_test_ran_some_test_exist_some_not'),
+                test.test_regrtest.ArgsTestCase('test_no_tests_ran'),
+                test.test_regrtest.ArgsTestCase('test_no_tests_ran_multiple_tests_nonexistent'),
+                test.test_regrtest.ArgsTestCase('test_random'),
+                test.test_regrtest.ArgsTestCase('test_rerun_fail'),
+                test.test_regrtest.ArgsTestCase('test_resources'),
+                test.test_regrtest.ArgsTestCase('test_slow_interrupted'),
+                test.test_regrtest.ArgsTestCase('test_slowest'),
+                test.test_regrtest.ArgsTestCase('test_wait'),
+                test.test_regrtest.ProgramsTestCase('test_module_autotest'),
+                test.test_regrtest.ProgramsTestCase('test_module_from_test_autotest'),
+                test.test_regrtest.ProgramsTestCase('test_module_regrtest'),
+                test.test_regrtest.ProgramsTestCase('test_module_test'),
+                test.test_regrtest.ProgramsTestCase('test_pcbuild_rt'),
+                test.test_regrtest.ProgramsTestCase('test_script_autotest'),
+                test.test_regrtest.ProgramsTestCase('test_script_regrtest'),
+                test.test_regrtest.ProgramsTestCase('test_tools_buildbot_test'),
+                test.test_regrtest.ProgramsTestCase('test_tools_script_run_tests'),
+            ]
+
+        return generate_suite(tests, failing_tests, skip_tests)
+
+    else:
+        return tests
+
+run_test(__name__)