Skip to content

Commit 8b1c977

Browse files
authored
Use default encoding as preferred encoding (#1528)
* Use default encoding as preferred encoding * Add test * Use UTF-8 on non-Windows * Update test * Add missing import
1 parent 0df498f commit 8b1c977

File tree

3 files changed

+60
-50
lines changed

3 files changed

+60
-50
lines changed

Src/IronPython/Modules/_io.cs

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1991,12 +1991,7 @@ public void __init__(
19911991
throw PythonOps.ValueError(string.Format("illegal newline value: " + newline));
19921992
}
19931993

1994-
if (encoding == null) {
1995-
encoding = PythonLocale.PreferredEncoding;
1996-
if (encoding == "") {
1997-
encoding = "ascii";
1998-
}
1999-
}
1994+
encoding ??= PythonLocale.PreferredEncoding;
20001995

20011996
if (errors == null) {
20021997
errors = "strict";

Src/IronPython/Modules/_locale.cs

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,13 @@
66
using System.Globalization;
77
using System.Runtime.CompilerServices;
88
using System.Runtime.InteropServices;
9+
using System.Text;
10+
911
using IronPython.Runtime;
1012
using IronPython.Runtime.Exceptions;
1113
using IronPython.Runtime.Types;
12-
using Microsoft.Scripting;
14+
1315
using Microsoft.Scripting.Runtime;
14-
using Microsoft.Scripting.Utils;
1516

1617
[assembly: PythonModule("_locale", typeof(IronPython.Modules.PythonLocale))]
1718
namespace IronPython.Modules {
@@ -40,16 +41,13 @@ internal static void EnsureLocaleInitialized(PythonContext context) {
4041
public const int LC_NUMERIC = (int)LocaleCategories.Numeric;
4142
public const int LC_TIME = (int)LocaleCategories.Time;
4243

43-
internal static string PreferredEncoding {
44-
get {
45-
return "cp" + CultureInfo.CurrentCulture.TextInfo.ANSICodePage.ToString();
46-
}
47-
}
44+
internal static string PreferredEncoding
45+
=> RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "cp" + Encoding.GetEncoding(0).CodePage.ToString() : "UTF-8";
4846

4947
[Documentation("gets the default locale tuple")]
50-
public static object _getdefaultlocale() {
48+
public static object _getdefaultlocale() {
5149
return PythonTuple.MakeTuple(
52-
GetDefaultLocale(),
50+
GetDefaultLocale(),
5351
PreferredEncoding
5452
);
5553
}
@@ -73,12 +71,12 @@ public static object localeconv(CodeContext/*!*/ context) {
7371
7472
If locale is None then the current setting is returned.
7573
")]
76-
public static object setlocale(CodeContext/*!*/ context, int category, string locale=null) {
74+
public static object setlocale(CodeContext/*!*/ context, int category, string locale = null) {
7775
LocaleInfo li = GetLocaleInfo(context);
7876
if (locale == null) {
7977
return li.GetLocale(context, category);
8078
}
81-
// An empty string specifies the users default settings.
79+
// An empty string specifies the user's default settings.
8280
if (locale == "") {
8381
locale = GetDefaultLocale();
8482
}
@@ -137,9 +135,9 @@ public CultureInfo Collate {
137135

138136
public CultureInfo CType {
139137
get { return _context.CTypeCulture; }
140-
set { _context.CTypeCulture= value; }
138+
set { _context.CTypeCulture = value; }
141139
}
142-
140+
143141
public CultureInfo Time {
144142
get { return _context.TimeCulture; }
145143
set { _context.TimeCulture = value; }
@@ -149,7 +147,7 @@ public CultureInfo Monetary {
149147
get { return _context.MonetaryCulture; }
150148
set { _context.MonetaryCulture = value; }
151149
}
152-
150+
153151
public CultureInfo Numeric {
154152
get { return _context.NumericCulture; }
155153
set { _context.NumericCulture = value; }
@@ -176,9 +174,9 @@ public string SetLocale(CodeContext/*!*/ context, int category, string locale) {
176174
case LocaleCategories.Collate:
177175
return CultureToName(Collate = LocaleToCulture(context, locale));
178176
case LocaleCategories.CType:
179-
return CultureToName(CType = LocaleToCulture(context, locale));
177+
return CultureToName(CType = LocaleToCulture(context, locale));
180178
case LocaleCategories.Time:
181-
return CultureToName(Time = LocaleToCulture(context, locale));
179+
return CultureToName(Time = LocaleToCulture(context, locale));
182180
case LocaleCategories.Monetary:
183181
Monetary = LocaleToCulture(context, locale);
184182
conv = null;
@@ -225,7 +223,7 @@ public string CultureToName(CultureInfo culture) {
225223
if (culture == PythonContext.CCulture) {
226224
return "C";
227225
}
228-
226+
229227
return culture.Name.Replace('-', '_');
230228
}
231229

@@ -291,7 +289,7 @@ private static PythonList GroupsToList(int[] groups) {
291289
EnsureLocaleInitialized(context.LanguageContext);
292290

293291
return (LocaleInfo)context.LanguageContext.GetModuleState(_localeKey);
294-
}
292+
}
295293

296294
private static PythonType _localeerror(CodeContext/*!*/ context) {
297295
return (PythonType)context.LanguageContext.GetModuleState("_localeerror");

Tests/modules/system_related/test__locale.py

Lines changed: 43 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import _random
77
import unittest
88

9-
from iptest import is_cli, is_netcoreapp, is_posix, run_test
9+
from iptest import is_cli, is_mono, is_netcoreapp, is_posix, run_test, skipUnlessIronPython
1010

1111
class _LocaleTest(unittest.TestCase):
1212

@@ -27,37 +27,37 @@ def test_getdefaultlocale(self):
2727
result1 = _locale._getdefaultlocale()
2828
result2 = _locale._getdefaultlocale()
2929
self.assertEqual(result1,result2)
30-
30+
3131
def test_localeconv(self):
3232
result = _locale.localeconv()
3333
self.assertTrue(result != None, "The method does not return the correct value")
3434
self.assertTrue(len(result)>=14, "The elements in the sequence are not enough")
35-
35+
3636
def test_strxfrm(self):
3737
str1 = "this is a test !"
3838
str2 = _locale.strxfrm(str1)
3939
str1.__eq__(str2)
40-
40+
4141
str1 = "this is a test&^%%^$%$ !"
4242
str2 = _locale.strxfrm(str1)
4343
str1.__eq__(str2)
44-
44+
4545
str1 = "this \" \"is a test&^%%^$%$ !"
4646
str2 = _locale.strxfrm(str1)
4747
str1.__eq__(str2)
48-
48+
4949
str1 = "123456789"
5050
str2 = _locale.strxfrm(str1)
5151
str1.__eq__(str2)
52-
52+
5353
#argument is int type
5454
i = 12
5555
self.assertRaises(TypeError,_locale.strxfrm,i)
56-
56+
5757
#argument is random type
5858
obj = _random.Random()
5959
self.assertRaises(TypeError,_locale.strxfrm,obj)
60-
60+
6161
def collate(self,str1,str2,result):
6262
if result ==0:
6363
self.assertTrue(_locale.strcoll(str1,str2)==0, "expected that collating \" %r \" and \" %r \" result should greater than zero" % (str1, str2))
@@ -72,48 +72,48 @@ def validcollate(self):
7272
str1 ="This is a test"
7373
str2 ="This is a test"
7474
self.collate(str1,str2,result)
75-
75+
7676
str1 ="@$%$%+_)(*"
7777
str2 ="@$%$%+_)(*"
7878
self.collate(str1,str2,result)
79-
79+
8080
str1 ="123This is a test"
8181
str2 ="123This is a test"
8282
self.collate(str1,str2,result)
83-
83+
8484
# str1 is greater than str2
8585
result =1
8686
str1 = "this is a test"
8787
self.collate(str1,str2,result)
88-
88+
8989
str1 = "234 this is a test"
9090
str2 = "123 this is a test"
9191
self.collate(str1,str2,result)
92-
92+
9393
str2 = "@^@%^#& this is a test"
9494
str2 = "#^@%^#& this is a test"
9595
self.collate(str1,str2,result)
96-
96+
9797
# str1 is less than str2
9898
result = -1
9999
str1 = "this is a test"
100100
str2 = "This Is a Test"
101101
self.collate(str1,str2,result)
102-
102+
103103
str1 = "This is a test123"
104104
str2 = "This is a test456"
105105
self.collate(str1,str2,result)
106-
106+
107107
str1 = "#$%This is a test"
108108
str2 = "@#$This is a test"
109109
self.collate(str1,str2,result)
110-
110+
111111
# an argument is int
112112
str2 = 3
113113
self.assertRaises(TypeError,_locale.strcoll,str1,str2)
114114
str2 = -1
115115
self.assertRaises(TypeError,_locale.strcoll,str1,str2)
116-
116+
117117
# an argument is Random type
118118
str2 = _random.Random()
119119
self.assertRaises(TypeError,_locale.strcoll,str1,str2)
@@ -122,7 +122,7 @@ def validcollate(self):
122122
def test_strcoll(self):
123123
_locale.setlocale(_locale.LC_COLLATE,"English")
124124
self.validcollate()
125-
125+
126126
_locale.setlocale(_locale.LC_COLLATE,"French")
127127
self.validcollate()
128128

@@ -135,14 +135,13 @@ def test_setlocale(self):
135135
_locale.LC_NUMERIC,
136136
_locale.LC_TIME,
137137
]
138-
138+
139139
for c in c_list:
140140
resultLocale = None
141141
_locale.setlocale(c,"English")
142142
resultLocale = _locale.setlocale(c)
143143
self.assertEqual(resultLocale,"English_United States.1252")
144-
145-
144+
146145
for c in c_list:
147146
resultLocale = None
148147
_locale.setlocale(c,"French")
@@ -155,13 +154,13 @@ def test_setlocale_negative(self):
155154
c= _locale.LC_ALL
156155
locale =''
157156
_locale.setlocale(c,locale)
158-
157+
159158
#the locale is None
160159
locale = _locale.setlocale(c)
161160
resultLocale =_locale.setlocale(c)
162161
self.assertEqual(locale,resultLocale)
163-
164-
#set Numeric as a unknown locale,should thorw a _locale.Error
162+
163+
#set Numeric as a unknown locale,should throw a _locale.Error
165164
locale ="11-22"
166165
self.assertRaises(_locale.Error,_locale.setlocale,c,locale)
167166
locale ="@^#^&%"
@@ -183,4 +182,22 @@ def test_bad_category(self):
183182
self.assertRaises(TypeError, _locale.setlocale, 'LC_NUMERIC', 'en_US.UTF8')
184183
self.assertRaises(TypeError, _locale.setlocale, 'LC_NUMERIC', None)
185184

185+
@skipUnlessIronPython()
186+
def test__getdefaultlocale(self):
187+
import System
188+
current_culture = System.Globalization.CultureInfo.CurrentCulture
189+
try:
190+
# https://github.com/IronLanguages/ironpython3/issues/1527
191+
for x in range(26):
192+
for y in range(26):
193+
try:
194+
culture_info = System.Globalization.CultureInfo("en-{}{}".format(chr(x + ord('A')), chr(y + ord('A'))))
195+
except ValueError:
196+
self.assertTrue(is_mono)
197+
else:
198+
System.Globalization.CultureInfo.CurrentCulture = culture_info
199+
self.assertNotEqual(_locale._getdefaultlocale()[1], "cp0")
200+
finally:
201+
System.Globalization.CultureInfo.CurrentCulture = current_culture
202+
186203
run_test(__name__)

0 commit comments

Comments
 (0)