Patch wx overriding the correct python locale (#12214)

The thread executing this did not have the correct locale set by NVDA through languageHandler.setLanguage. This is due to the latest wxPython incorrectly overriding the locale with one not supported by python.

The Windows/System language option should also call locale.setlocale in setLanguage and the locale needs to be reset after wx changes it.

source/core.py

@@ -449,6 +449,9 @@ def handlePowerStatusChange(self):
 			locale.Init(wxLang.Language)
 		except:
 			log.error("Failed to initialize wx locale",exc_info=True)
+		finally:
+			# Revert wx's changes to the python locale
+			languageHandler.setLocale(languageHandler.curLang)
 
 	log.debug("Initializing garbageHandler")
 	garbageHandler.initialize()

source/languageHandler.py

@@ -13,6 +13,7 @@
 import locale
 import gettext
 import globalVars
+from logHandler import log
 
 #a few Windows locale constants
 LOCALE_SLANGUAGE=0x2
@@ -148,37 +149,119 @@ def getWindowsLanguage():
 			localeName="en"
 	return localeName
 
-def setLanguage(lang):
+
+def setLanguage(lang: str) -> None:
+	'''
+	Sets the following using `lang` such as "en", "ru_RU", or "es-ES". Use "Windows" to use the system locale
+	 - the windows locale for the thread (fallback to system locale)
+	 - the translation service (fallback to English)
+	 - languageHandler.curLang (match the translation service)
+	 - the python locale for the thread (match the translation service, fallback to system default)
+	'''
 	global curLang
-	try:
-		if lang=="Windows":
-			localeName=getWindowsLanguage()
-			trans=gettext.translation('nvda',localedir='locale',languages=[localeName])
-			curLang=localeName
-		else:
-			trans=gettext.translation("nvda", localedir="locale", languages=[lang])
-			curLang=lang
-			localeChanged=False
-			#Try setting Python's locale to lang
-			try:
-				locale.setlocale(locale.LC_ALL,lang)
-				localeChanged=True
-			except:
-				pass
-			if not localeChanged and '_' in lang:
-				#Python couldn'tsupport the language_country locale, just try language.
-				try:
-					locale.setlocale(locale.LC_ALL,lang.split('_')[0])
-				except:
-					pass
-			#Set the windows locale for this thread (NVDA core) to this locale.
-			LCID=localeNameToWindowsLCID(lang)
+	if lang == "Windows":
+		localeName = getWindowsLanguage()
+	else:
+		localeName = lang
+		# Set the windows locale for this thread (NVDA core) to this locale.
+		try:
+			LCID = localeNameToWindowsLCID(lang)
 			ctypes.windll.kernel32.SetThreadLocale(LCID)
+		except IOError:
+			log.debugWarning(f"couldn't set windows thread locale to {lang}")
+
+	try:
+		trans = gettext.translation("nvda", localedir="locale", languages=[localeName])
+		curLang = localeName
 	except IOError:
-		trans=gettext.translation("nvda",fallback=True)
-		curLang="en"
+		log.debugWarning(f"couldn't set the translation service locale to {localeName}")
+		trans = gettext.translation("nvda", fallback=True)
+		curLang = "en"
+
 	# #9207: Python 3.8 adds gettext.pgettext, so add it to the built-in namespace.
 	trans.install(names=["pgettext"])
+	setLocale(curLang)
+
+
+def setLocale(localeName: str) -> None:
+	'''
+	Set python's locale using a `localeName` such as "en", "ru_RU", or "es-ES".
+	Will fallback on `curLang` if it cannot be set and finally fallback to the system locale.
+	'''
+
+	r'''
+	Python 3.8's locale system allows you to set locales that you cannot get
+	so we must test for both ValueErrors and locale.Errors
+
+	>>> import locale
+	>>> locale.setlocale(locale.LC_ALL, 'foobar')
+	Traceback (most recent call last):
+	File "<stdin>", line 1, in <module>
+	File "Python38-32\lib\locale.py", line 608, in setlocale
+		return _setlocale(category, locale)
+	locale.Error: unsupported locale setting
+	>>> locale.setlocale(locale.LC_ALL, 'en-GB')
+	'en-GB'
+	>>> locale.getlocale()
+	Traceback (most recent call last):
+	File "<stdin>", line 1, in <module>
+	File "Python38-32\lib\locale.py", line 591, in getlocale
+		return _parse_localename(localename)
+	File "Python38-32\lib\locale.py", line 499, in _parse_localename
+		raise ValueError('unknown locale: %s' % localename)
+	ValueError: unknown locale: en-GB
+	'''
+	originalLocaleName = localeName
+	# Try setting Python's locale to localeName
+	try:
+		locale.setlocale(locale.LC_ALL, localeName)
+		locale.getlocale()
+		log.debug(f"set python locale to {localeName}")
+		return
+	except locale.Error:
+		log.debugWarning(f"python locale {localeName} could not be set")
+	except ValueError:
+		log.debugWarning(f"python locale {localeName} could not be retrieved with getlocale")
+
+	if '-' in localeName:
+		# Python couldn't support the language-country locale, try language_country.
+		try:
+			localeName = localeName.replace('-', '_')
+			locale.setlocale(locale.LC_ALL, localeName)
+			locale.getlocale()
+			log.debug(f"set python locale to {localeName}")
+			return
+		except locale.Error:
+			log.debugWarning(f"python locale {localeName} could not be set")
+		except ValueError:
+			log.debugWarning(f"python locale {localeName} could not be retrieved with getlocale")
+
+	if '_' in localeName:
+		# Python couldn't support the language_country locale, just try language.
+		try:
+			localeName = localeName.split('_')[0]
+			locale.setlocale(locale.LC_ALL, localeName)
+			locale.getlocale()
+			log.debug(f"set python locale to {localeName}")
+			return
+		except locale.Error:
+			log.debugWarning(f"python locale {localeName} could not be set")
+		except ValueError:
+			log.debugWarning(f"python locale {localeName} could not be retrieved with getlocale")
+
+	try:
+		locale.getlocale()
+	except ValueError:
+		# as the locale may have been changed to something that getlocale() couldn't retrieve
+		# reset to default locale
+		if originalLocaleName == curLang:
+			# reset to system locale default if we can't set the current lang's locale
+			locale.setlocale(locale.LC_ALL, "")
+			log.debugWarning(f"set python locale to system default")
+		else:
+			log.debugWarning(f"setting python locale to the current language {curLang}")
+			# fallback and try to reset the locale to the current lang
+			setLocale(curLang)
 
 
 def getLanguage() -> str:
@@ -316,5 +399,9 @@ def normalizeLanguage(lang):
 	134:'qut',
 	135:'rw',
 	136:'wo',
-	140:'gbz'
+	140: 'gbz',
+	1170: 'ckb',
+	1109: 'my',
+	1143: 'so',
+	9242: 'sr',
 }

tests/unit/test_languageHandler.py

@@ -1,20 +1,32 @@
-#tests/unit/test_languageHandler.py
-#A part of NonVisual Desktop Access (NVDA)
-#This file is covered by the GNU General Public License.
-#See the file COPYING for more details.
-#Copyright (C) 2017 NV Access Limited
+# A part of NonVisual Desktop Access (NVDA)
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+# Copyright (C) 2017-2021 NV Access Limited
 
 """Unit tests for the languageHandler module.
 """
 
 import unittest
 import languageHandler
-from languageHandler import LCID_NONE
+from languageHandler import LCID_NONE, windowsPrimaryLCIDsToLocaleNames
+import locale
+import ctypes
 
 LCID_ENGLISH_US = 0x0409
+UNSUPPORTED_PYTHON_LOCALES = {
+	"an",
+	"ckb",
+	"kmr",
+	"mn",
+	"my",
+	"ne",
+	"so",
+}
+TRANSLATABLE_LANGS = set(l[0] for l in languageHandler.getAvailableLanguages()) - {"Windows"}
+WINDOWS_LANGS = set(locale.windows_locale.values()).union(windowsPrimaryLCIDsToLocaleNames.values())
 
-class TestLocaleNameToWindowsLCID(unittest.TestCase):
 
+class TestLocaleNameToWindowsLCID(unittest.TestCase):
 	def test_knownLocale(self):
 		lcid = languageHandler.localeNameToWindowsLCID("en")
 		self.assertEqual(lcid, LCID_ENGLISH_US)
@@ -31,3 +43,156 @@ def test_nonStandardLocale(self):
 	def test_invalidLocale(self):
 		lcid = languageHandler.localeNameToWindowsLCID("zzzz")
 		self.assertEqual(lcid, LCID_NONE)
+
+
+class Test_languageHandler_setLocale(unittest.TestCase):
+	"""Tests for the function languageHandler.setLocale"""
+
+	SUPPORTED_LOCALES = [("en", "en_US"), ("fa-IR", "fa_IR"), ("an-ES", "an_ES")]
+
+	def setUp(self):
+		"""
+		`setLocale` doesn't change `languageHandler.curLang`, so reset the locale using `setLanguage` to
+		the current language for each test.
+		"""
+		languageHandler.setLanguage(languageHandler.curLang)
+
+	@classmethod
+	def tearDownClass(cls):
+		"""
+		`setLocale` doesn't change `languageHandler.curLang`, so reset the locale using `setLanguage` to
+		the current language so the tests can continue normally.
+		"""
+		languageHandler.setLanguage(languageHandler.curLang)
+
+	def test_SupportedLocale_LocaleIsSet(self):
+		"""
+		Tests several locale formats that should result in an expected python locale being set.
+		"""
+		for localeName in self.SUPPORTED_LOCALES:
+			with self.subTest(localeName=localeName):
+				languageHandler.setLocale(localeName[0])
+				self.assertEqual(locale.getlocale()[0], localeName[1])
+
+	def test_PythonUnsupportedLocale_LocaleUnchanged(self):
+		"""
+		Tests several locale formats that python doesn't support which will result in a return to the
+		current locale
+		"""
+		original_locale = locale.getlocale()
+		for localeName in UNSUPPORTED_PYTHON_LOCALES:
+			with self.subTest(localeName=localeName):
+				languageHandler.setLocale(localeName)
+				self.assertEqual(locale.getlocale(), original_locale)
+
+	def test_NVDASupportedAndPythonSupportedLocale_LanguageCodeMatches(self):
+		"""
+	 	Tests all the translatable languages that NVDA shows in the user preferences
+		excludes the locales that python doesn't support, as the expected behaviour is different.
+		"""
+		for localeName in TRANSLATABLE_LANGS - UNSUPPORTED_PYTHON_LOCALES:
+			with self.subTest(localeName=localeName):
+				languageHandler.setLocale(localeName)
+				current_locale = locale.getlocale()
+
+				if localeName == "uk":
+					self.assertEqual(current_locale[0], "English_United Kingdom")
+				else:
+					pythonLang = current_locale[0].split("_")[0]
+					langOnly = localeName.split("_")[0]
+					self.assertEqual(
+						langOnly,
+						pythonLang,
+						f"full values: {localeName} {current_locale[0]}",
+					)
+
+	def test_WindowsLang_LocaleCanBeRetrieved(self):
+		"""
+		We don't know whether python supports a specific windows locale so just ensure locale isn't
+		broken after testing these values.
+		"""
+		for localeName in WINDOWS_LANGS:
+			with self.subTest(localeName=localeName):
+				languageHandler.setLocale(localeName)
+				locale.getlocale()
+
+
+class Test_LanguageHandler_SetLanguage(unittest.TestCase):
+	"""Tests for the function languageHandler.setLanguage"""
+
+	UNSUPPORTED_WIN_LANGUAGES = ["an", "kmr"]
+
+	def tearDown(self):
+		"""
+		Resets the language to whatever it was before the testing suite begun.
+		"""
+		languageHandler.setLanguage(self._prevLang)
+
+	def __init__(self, *args, **kwargs):
+		self._prevLang = languageHandler.getLanguage()
+
+		ctypes.windll.kernel32.SetThreadLocale(0)
+		defaultThreadLocale = ctypes.windll.kernel32.GetThreadLocale()
+		self._defaultThreadLocaleName = languageHandler.windowsLCIDToLocaleName(
+			defaultThreadLocale
+		)
+
+		locale.setlocale(locale.LC_ALL, "")
+		self._defaultPythonLocale = locale.getlocale()
+
+		languageHandler.setLanguage(self._prevLang)
+		super().__init__(*args, **kwargs)
+
+	def test_NVDASupportedLanguages_LanguageIsSetCorrectly(self):
+		"""
+		Tests languageHandler.setLanguage, using all NVDA supported languages, which should do the following:
+		- set the translation service and languageHandler.curLang
+		- set the windows locale for the thread (fallback to system default)
+		- set the python locale for the thread (match the translation service, fallback to system default)
+		"""
+		for localeName in TRANSLATABLE_LANGS:
+			with self.subTest(localeName=localeName):
+				langOnly = localeName.split("_")[0]
+				languageHandler.setLanguage(localeName)
+				# check curLang/translation service is set
+				self.assertEqual(languageHandler.curLang, localeName)
+
+				# check Windows thread is set
+				threadLocale = ctypes.windll.kernel32.GetThreadLocale()
+				threadLocaleName = languageHandler.windowsLCIDToLocaleName(threadLocale)
+				threadLocaleLang = threadLocaleName.split("_")[0]
+				if localeName in self.UNSUPPORTED_WIN_LANGUAGES:
+					# our translatable locale isn't supported by windows
+					# check that the system locale is unchanged
+					self.assertEqual(self._defaultThreadLocaleName, threadLocaleName)
+				else:
+					# check that the language codes are correctly set for the thread
+					self.assertEqual(
+						langOnly,
+						threadLocaleLang,
+						f"full values: {localeName} {threadLocaleName}",
+					)
+
+				# check that the python locale is set
+				python_locale = locale.getlocale()
+				if localeName in UNSUPPORTED_PYTHON_LOCALES:
+					# our translatable locale isn't supported by python
+					# check that the system locale is unchanged
+					self.assertEqual(self._defaultPythonLocale, python_locale)
+				elif localeName == "uk":
+					self.assertEqual(python_locale[0], "English_United Kingdom")
+				else:
+					# check that the language codes are correctly set for python
+					pythonLang = python_locale[0].split("_")[0]
+					self.assertEqual(
+						langOnly, pythonLang, f"full values: {localeName} {python_locale}"
+					)
+
+	def test_WindowsLanguages_NoErrorsThrown(self):
+		"""
+		We don't know whether python or our translator system supports a specific windows locale
+		so just ensure the setLanguage process doesn't fail.
+		"""
+		for localeName in WINDOWS_LANGS:
+			with self.subTest(localeName=localeName):
+				languageHandler.setLanguage(localeName)

user_docs/en/changes.t2t

@@ -39,6 +39,7 @@ What's New in NVDA
 - Fix graphical bugs such as missing elements when using NVDA with a right-to-left layout. (#8859)
 - Respect the GUI layout direction based on the NVDA language, not the system locale. (#638)
    - known issue for right-to-left languages: the right border of groupings clips with labels/controls. (#12181)
+- The python locale is set to match the language selected in preferences consistently, and will occur when using the default language. (#12214)
 
 
 == Changes for Developers ==