Merge branch 'translation-lookup-fixes' of https://github.com/vslavik/wxWidgets

Restore correct translation language lookup taking into account message
IDs language correctly when it is present in the preferred UI languages
but is not the first language in them and other similar corner cases.

Also add unit tests verifying that this code logic, already changed many
times in the past, does behave correctly now.

See #24297.
This commit is contained in:
Vadim Zeitlin 2024-02-13 21:34:00 +01:00
commit 01ffca369c
13 changed files with 235 additions and 83 deletions

View file

@ -106,6 +106,8 @@ if(wxUSE_XML)
endif()
set(TEST_DATA
intl/en_GB/internat.mo
intl/en_GB/internat.po
intl/fr/internat.mo
intl/fr/internat.po
intl/ja/internat.mo

View file

@ -51,6 +51,12 @@ wxWidgets programs.
default value if it's not a number, so that e.g. setting it to "yes"
suppresses all GTK diagnostics while setting it to 16 only suppresses
GTK warning messages.}
@itemdef{WXLANGUAGE,
This variable can be set to override OS setting of preferred languages
and make wxUILocale::GetPreferredUILanguages() return the set list
instead. The format is same as GNU's <a
href="https://www.gnu.org/software/gettext/manual/html_node/The-LANGUAGE-variable.html">LANGUAGE</a>
variable: a colon-separated list of language codes.}
*/
@see wxSystemOptions

View file

@ -160,7 +160,7 @@ public:
// add catalog for the given domain returning true if it could be found by
// wxTranslationsLoader
bool AddAvailableCatalog(const wxString& domain);
bool AddAvailableCatalog(const wxString& domain, wxLanguage msgIdLanguage = wxLANGUAGE_ENGLISH_US);
// add standard wxWidgets catalog ("wxstd")
bool AddStdCatalog();
@ -193,6 +193,15 @@ public:
static const wxString& GetUntranslatedString(const wxString& str);
private:
enum class Translations
{
NotNeeded = -1,
NotFound = 0,
Found = 1
};
Translations DoAddCatalog(const wxString& domain, wxLanguage msgIdLanguage);
// perform loading of the catalog via m_loader
bool LoadCatalog(const wxString& domain, const wxString& lang);
@ -203,6 +212,8 @@ private:
static void SetNonOwned(wxTranslations *t);
friend class wxLocale;
wxString DoGetBestAvailableTranslation(const wxString& domain, const wxString& additionalAvailableLanguage);
private:
wxString m_lang;
wxTranslationsLoader *m_loader;

View file

@ -85,6 +85,11 @@ public:
translations offered to the user. To do this, pass the app's main
catalog as @a domain.
@note
The returned list does not include messages ID language, i.e. the
language (typically English) included in the source code. In the use
case described above, that language needs to be added manually.
@see GetBestTranslation()
*/
wxArrayString GetAvailableTranslations(const wxString& domain) const;
@ -97,6 +102,12 @@ public:
it simply returns the language set with SetLanguage() if it's available
or empty string otherwise.
@warning
This function does not consider messages ID language (typically
English) and can return inappropriate language if it is anywhere in
user's preferred languages list. Use GetBestTranslation() instead
unless you have very specific needs.
@since 3.3.0
*/
wxString GetBestAvailableTranslation(const wxString& domain);
@ -158,6 +169,16 @@ public:
All loaded catalogs will be used for message lookup by GetString() for
the current locale.
@param domain
The catalog domain to add.
@param msgIdLanguage
Specifies the language of "msgid" strings in source code
(i.e. arguments to GetString(), wxGetTranslation() and the _() macro).
It is used if AddCatalog() cannot find any catalog for current language:
if the language is same as source code language, then strings from source
code are used instead.
@return
@true if catalog was successfully loaded, @false otherwise, usually
because it wasn't found. Note that unlike AddCatalog() this
@ -167,9 +188,10 @@ public:
selected or system-default languages, but is not necessarily an
error if no translations are needed in the first place.
@since 3.3.0
@since 3.2.5
*/
bool AddAvailableCatalog(const wxString& domain);
bool AddAvailableCatalog(const wxString& domain,
wxLanguage msgIdLanguage = wxLANGUAGE_ENGLISH_US);
/**
Add a catalog for use with the current locale or fall back to the

View file

@ -116,14 +116,6 @@ void LogTraceLargeArray(const wxString& prefix, const wxArrayString& arr)
#endif // wxUSE_LOG_TRACE/!wxUSE_LOG_TRACE
// Use locale-based detection as a fallback
wxString GetPreferredUILanguageFallback(const wxArrayString& WXUNUSED(available))
{
const wxString lang = wxUILocale::GetLanguageCanonicalName(wxUILocale::GetSystemLocale());
wxLogTrace(TRACE_I18N, " - obtained best language from locale: %s", lang);
return lang;
}
wxString GetPreferredUILanguage(const wxArrayString& available)
{
wxVector<wxString> preferred = wxUILocale::GetPreferredUILanguages();
@ -168,7 +160,7 @@ wxString GetPreferredUILanguage(const wxArrayString& available)
if (!langNoMatchRegion.empty())
return langNoMatchRegion;
return GetPreferredUILanguageFallback(available);
return wxString();
}
} // anonymous namespace
@ -1316,65 +1308,50 @@ bool wxTranslations::AddStdCatalog()
// the name without the version if it's not found, as message catalogs
// typically won't have the version in their names under non-Unix platforms
// (i.e. where they're not installed by our own "make install").
if ( AddAvailableCatalog("wxstd-" wxSTRINGIZE(wxMAJOR_VERSION) "." wxSTRINGIZE(wxMINOR_VERSION)) )
return true;
wxString domain("wxstd-" wxSTRINGIZE(wxMAJOR_VERSION) "." wxSTRINGIZE(wxMINOR_VERSION));
if ( GetBestAvailableTranslation(domain).empty() )
domain = wxS("wxstd");
if ( AddCatalog(wxS("wxstd")) )
return true;
return false;
return AddCatalog(domain);
}
bool wxTranslations::AddAvailableCatalog(const wxString& domain)
bool wxTranslations::AddAvailableCatalog(const wxString& domain, wxLanguage msgIdLanguage)
{
const wxString domain_lang = GetBestAvailableTranslation(domain);
return DoAddCatalog(domain, msgIdLanguage) == Translations::Found;
}
bool wxTranslations::AddCatalog(const wxString& domain, wxLanguage msgIdLanguage)
{
return DoAddCatalog(domain, msgIdLanguage) != Translations::NotFound;
}
wxTranslations::Translations wxTranslations::DoAddCatalog(const wxString& domain,
wxLanguage msgIdLanguage)
{
const wxString msgIdLang = wxUILocale::GetLanguageCanonicalName(msgIdLanguage);
const wxString domain_lang = GetBestTranslation(domain, msgIdLang);
if ( domain_lang.empty() )
{
wxLogTrace(TRACE_I18N,
wxS("no suitable translation for domain '%s' found"),
domain);
return false;
return Translations::NotFound;
}
return LoadCatalog(domain, domain_lang);
}
bool wxTranslations::AddCatalog(const wxString& domain,
wxLanguage msgIdLanguage)
{
if ( AddAvailableCatalog(domain) )
return true;
const wxString msgIdLang = wxUILocale::GetLanguageCanonicalName(msgIdLanguage);
// Check if the original strings can be used directly.
bool canUseUntranslated = false;
if ( m_lang.empty() )
{
// If we are using the default language, check if the message ID
// language is acceptable for this system.
const wxString domain_lang = GetBestTranslation(domain, msgIdLang);
if ( msgIdLang == domain_lang )
canUseUntranslated = true;
}
else // But if we have a fixed language, we should just check it instead.
{
// Consider message IDs for another region using the same language
// acceptable.
if ( msgIdLang.BeforeFirst('_') == m_lang.BeforeFirst('_') )
canUseUntranslated = true;
}
if ( canUseUntranslated )
if ( LoadCatalog(domain, domain_lang) )
{
wxLogTrace(TRACE_I18N,
wxS("not using translations for domain '%s' with msgid language '%s'"),
domain, msgIdLang);
return true;
wxS("adding '%s' translation for domain '%s' (msgid language '%s')"),
domain_lang, domain, msgIdLang);
return Translations::Found;
}
return false;
// LoadCatalog() failed, but GetBestTranslation() returned non-empty language.
// That must mean that msgIdLanguage was used.
wxLogTrace(TRACE_I18N,
wxS("not using translations for domain '%s' with msgid language '%s'"),
domain, msgIdLang);
return Translations::NotNeeded;
}
@ -1452,23 +1429,19 @@ wxString wxTranslations::GetBestTranslation(const wxString& domain,
wxString wxTranslations::GetBestTranslation(const wxString& domain,
const wxString& msgIdLanguage)
{
wxString lang = GetBestAvailableTranslation(domain);
// Determine the best language, including the msgId language, which is always
// available because it is present in the code:
wxString lang = DoGetBestAvailableTranslation(domain, msgIdLanguage);
if ( lang.empty() )
{
wxArrayString available;
available.push_back(msgIdLanguage);
available.push_back(msgIdLanguage.BeforeFirst('_'));
lang = GetPreferredUILanguage(available);
if ( lang.empty() )
{
wxLogTrace(TRACE_I18N,
"no available language for domain '%s'", domain);
}
else
{
wxLogTrace(TRACE_I18N,
"using message ID language '%s' for domain '%s'", lang);
}
wxLogTrace(TRACE_I18N,
"no available language for domain '%s'", domain);
}
else if ( lang == msgIdLanguage || lang == msgIdLanguage.BeforeFirst('_') )
{
wxLogTrace(TRACE_I18N,
"using message ID language '%s' for domain '%s'", lang, domain);
}
return lang;
@ -1476,7 +1449,21 @@ wxString wxTranslations::GetBestTranslation(const wxString& domain,
wxString wxTranslations::GetBestAvailableTranslation(const wxString& domain)
{
const wxArrayString available(GetAvailableTranslations(domain));
// Determine the best language from the ones with actual translation file:
// As this function never considers the language of the original messages as being
// available, pass empty string as message ID language to the helper function.
return DoGetBestAvailableTranslation(domain, wxString());
}
wxString wxTranslations::DoGetBestAvailableTranslation(const wxString& domain, const wxString& additionalAvailableLanguage)
{
wxArrayString available(GetAvailableTranslations(domain));
if ( !additionalAvailableLanguage.empty() )
{
available.push_back(additionalAvailableLanguage);
available.push_back(additionalAvailableLanguage.BeforeFirst('_'));
}
if ( !m_lang.empty() )
{
wxLogTrace(TRACE_I18N,

View file

@ -24,6 +24,9 @@
#include "wx/arrstr.h"
#include "wx/intl.h"
#include "wx/log.h"
#include "wx/tokenzr.h"
#include "wx/utils.h"
#ifndef __WINDOWS__
#include "wx/language.h"
@ -31,6 +34,8 @@
#include "wx/private/uilocale.h"
#define TRACE_I18N wxS("i18n")
// ----------------------------------------------------------------------------
// helper functions
// ----------------------------------------------------------------------------
@ -678,7 +683,7 @@ int wxUILocale::GetSystemLanguage()
{
const wxLanguageInfos& languagesDB = wxGetLanguageInfos();
size_t count = languagesDB.size();
wxVector<wxString> preferred = wxUILocaleImpl::GetPreferredUILanguages();
wxVector<wxString> preferred = wxUILocale::GetPreferredUILanguages();
for (wxVector<wxString>::const_iterator j = preferred.begin();
j != preferred.end();
@ -743,6 +748,29 @@ int wxUILocale::GetSystemLocale()
/* static */
wxVector<wxString> wxUILocale::GetPreferredUILanguages()
{
// The WXLANGUAGE variable may contain a colon separated list of language
// codes in the order of preference. It is modelled after GNU's LANGUAGE:
// http://www.gnu.org/software/gettext/manual/html_node/The-LANGUAGE-variable.html
wxString languageFromEnv;
if (wxGetEnv("WXLANGUAGE", &languageFromEnv) && !languageFromEnv.empty())
{
wxVector<wxString> preferred;
wxStringTokenizer tknzr(languageFromEnv, ":");
while (tknzr.HasMoreTokens())
{
const wxString tok = tknzr.GetNextToken();
if (const wxLanguageInfo* li = wxUILocale::FindLanguageInfo(tok))
{
preferred.push_back(li->CanonicalName);
}
}
if (!preferred.empty())
{
wxLogTrace(TRACE_I18N, " - using languages override from WXLANGUAGE: '%s'", languageFromEnv);
return preferred;
}
}
return wxUILocaleImpl::GetPreferredUILanguages();
}

View file

@ -463,7 +463,7 @@ COND_MONOLITHIC_1___WXLIB_MONO_p = \
### Targets: ###
all: test$(EXEEXT) $(__test_drawing___depname) $(__test_drawingplugin___depname) $(__test_gui___depname) $(__test_gui_bundle___depname) $(__test_allheaders___depname) $(__test_allheaders_bundle___depname) data data-image-sample data-images fr ja
all: test$(EXEEXT) $(__test_drawing___depname) $(__test_drawingplugin___depname) $(__test_gui___depname) $(__test_gui_bundle___depname) $(__test_allheaders___depname) $(__test_allheaders_bundle___depname) data data-image-sample data-images en_GB fr ja
install:
@ -602,6 +602,18 @@ data-images:
esac; \
done
en_GB:
@mkdir -p ./intl/en_GB
@for f in internat.po internat.mo; do \
if test ! -f ./intl/en_GB/$$f -a ! -d ./intl/en_GB/$$f ; \
then x=yep ; \
else x=`find $(srcdir)/intl/en_GB/$$f -newer ./intl/en_GB/$$f -print` ; \
fi; \
case "$$x" in ?*) \
cp -pRf $(srcdir)/intl/en_GB/$$f ./intl/en_GB ;; \
esac; \
done
fr:
@mkdir -p ./intl/fr
@for f in internat.po internat.mo; do \
@ -1278,4 +1290,4 @@ failtest_allheaders:
@IF_GNU_MAKE@-include ./.deps/*.d
.PHONY: all install uninstall clean distclean test_gui_bundle \
test_allheaders_bundle data data-image-sample data-images fr ja
test_allheaders_bundle data data-image-sample data-images en_GB fr ja

Binary file not shown.

View file

@ -0,0 +1,18 @@
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: \n"
"POT-Creation-Date: 2003-10-04 23:10+0200\n"
"PO-Revision-Date: 2024-02-13 13:25+0100\n"
"Last-Translator: \n"
"Language-Team: \n"
"Language: en_GB\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"

View file

@ -19,6 +19,7 @@
#include "wx/intl.h"
#include "wx/uilocale.h"
#include "wx/scopeguard.h"
#include "wx/private/glibc.h"
@ -241,8 +242,9 @@ void IntlTestCase::IsAvailable()
TEST_CASE("wxTranslations::AddCatalog", "[translations]")
{
// We currently have translations for French and Japanese in this test
// directory, check that loading those succeeds but loading others doesn't.
// We currently have translations for British English, French and Japanese
// in this test directory, check that loading those succeeds but loading
// others doesn't.
wxFileTranslationsLoader::AddCatalogLookupPathPrefix("./intl");
const wxString domain("internat");
@ -252,11 +254,12 @@ TEST_CASE("wxTranslations::AddCatalog", "[translations]")
SECTION("All")
{
auto available = trans.GetAvailableTranslations(domain);
REQUIRE( available.size() == 2 );
REQUIRE( available.size() == 3 );
available.Sort();
CHECK( available[0] == "fr" );
CHECK( available[1] == "ja" );
CHECK( available[0] == "en_GB" );
CHECK( available[1] == "fr" );
CHECK( available[2] == "ja" );
}
SECTION("French")
@ -287,6 +290,60 @@ TEST_CASE("wxTranslations::AddCatalog", "[translations]")
}
}
TEST_CASE("wxTranslations::GetBestTranslation", "[translations]")
{
wxFileTranslationsLoader::AddCatalogLookupPathPrefix("./intl");
const wxString domain("internat");
wxTranslations trans;
wxON_BLOCK_EXIT1( wxUnsetEnv, "WXLANGUAGE" );
SECTION("ChooseLanguage")
{
// Simple case.
wxSetEnv("WXLANGUAGE", "fr:en");
CHECK( trans.GetBestTranslation(domain) == "fr" );
CHECK( trans.GetBestAvailableTranslation(domain) == "fr" );
// Choose 2nd language _and_ its base form.
wxSetEnv("WXLANGUAGE", "cs:fr_CA:en");
CHECK( trans.GetBestTranslation(domain) == "fr" );
CHECK( trans.GetBestAvailableTranslation(domain) == "fr" );
}
SECTION("EnglishHandling")
{
// Check that existing en_GB file isn't used for msgid language.
wxSetEnv("WXLANGUAGE", "en_US");
CHECK( trans.GetBestTranslation(domain) == "en" );
// GetBestAvailableTranslation() will wrongly return "en_GB", don't test that.
wxSetEnv("WXLANGUAGE", "es:en");
CHECK( trans.GetBestTranslation(domain) == "en" );
// GetBestAvailableTranslation() will wrongly return "en_GB", don't test that.
// And that it is used when it should be
wxSetEnv("WXLANGUAGE", "en_GB");
CHECK( trans.GetBestTranslation(domain) == "en_GB" );
CHECK( trans.GetBestAvailableTranslation(domain) == "en_GB" );
}
SECTION("DontSkipMsgidLanguage")
{
// Check that msgid language will be used if it's the best match.
wxSetEnv("WXLANGUAGE", "cs:en:fr");
CHECK( trans.GetBestTranslation(domain) == "en" );
// ...But won't be used if there's a suitable translation file.
wxSetEnv("WXLANGUAGE", "fr:en:cs");
CHECK( trans.GetBestTranslation(domain) == "fr" );
CHECK( trans.GetBestAvailableTranslation(domain) == "fr" );
}
}
TEST_CASE("wxLocale::Default", "[locale]")
{
const int langDef = wxUILocale::GetSystemLanguage();

View file

@ -512,7 +512,7 @@ $(OBJS):
### Targets: ###
all: $(OBJS)\test.exe $(__test_drawing___depname) $(__test_drawingplugin___depname) $(__test_gui___depname) $(__test_allheaders___depname) data data-image-sample data-images fr ja
all: $(OBJS)\test.exe $(__test_drawing___depname) $(__test_drawingplugin___depname) $(__test_gui___depname) $(__test_allheaders___depname) data data-image-sample data-images en_GB fr ja
clean:
-if exist $(OBJS)\*.o del $(OBJS)\*.o
@ -572,6 +572,10 @@ data-images:
if not exist image mkdir image
for %%f in (bitfields.bmp bitfields-alpha.bmp 8bpp-colorsused-large.bmp 8bpp-colorsused-negative.bmp rle4-delta-320x240.bmp rle8-delta-320x240.bmp rle8-delta-320x240-expected.bmp horse_grey.bmp horse_grey_flipped.bmp horse_rle4.bmp horse_rle4_flipped.bmp horse_rle8.bmp horse_rle8_flipped.bmp horse_bicubic_50x50.png horse_bicubic_100x100.png horse_bicubic_150x150.png horse_bicubic_300x300.png horse_bilinear_50x50.png horse_bilinear_100x100.png horse_bilinear_150x150.png horse_bilinear_300x300.png horse_box_average_50x50.png horse_box_average_100x100.png horse_box_average_150x150.png horse_box_average_300x300.png cross_bicubic_256x256.png cross_bilinear_256x256.png cross_box_average_256x256.png cross_nearest_neighb_256x256.png paste_input_background.png paste_input_black.png paste_input_overlay_transparent_border_opaque_square.png paste_input_overlay_transparent_border_semitransparent_circle.png paste_input_overlay_transparent_border_semitransparent_square.png paste_result_background_plus_circle_plus_square.png paste_result_background_plus_overlay_transparent_border_opaque_square.png paste_result_background_plus_overlay_transparent_border_semitransparent_square.png paste_result_no_background_square_over_circle.png wx.png toucan.png toucan_hue_0.538.png toucan_sat_-0.41.png toucan_bright_-0.259.png toucan_hsv_0.538_-0.41_-0.259.png toucan_light_46.png toucan_dis_240.png toucan_grey.png toucan_mono_255_255_255.png width-times-height-overflow.bmp width_height_32_bit_overflow.pgm bad_truncated.gif) do if not exist image\%%f copy .\image\%%f image
en_GB:
if not exist $(OBJS)\intl\en_GB mkdir $(OBJS)\intl\en_GB
for %%f in (internat.po internat.mo) do if not exist $(OBJS)\intl\en_GB\%%f copy .\intl\en_GB\%%f $(OBJS)\intl\en_GB
fr:
if not exist $(OBJS)\intl\fr mkdir $(OBJS)\intl\fr
for %%f in (internat.po internat.mo) do if not exist $(OBJS)\intl\fr\%%f copy .\intl\fr\%%f $(OBJS)\intl\fr
@ -1201,7 +1205,7 @@ $(OBJS)\test_allheaders_allheaders.o: ./allheaders.cpp
$(OBJS)\test_allheaders_testableframe.o: ./testableframe.cpp
$(CXX) -c -o $@ $(TEST_ALLHEADERS_CXXFLAGS) $(CPPDEPS) $<
.PHONY: all clean data data-image-sample data-images fr ja
.PHONY: all clean data data-image-sample data-images en_GB fr ja
SHELL := $(COMSPEC)

View file

@ -792,7 +792,7 @@ $(OBJS):
### Targets: ###
all: $(OBJS)\test.exe $(__test_drawing___depname) $(__test_drawingplugin___depname) $(__test_gui___depname) $(__test_allheaders___depname) data data-image-sample data-images fr ja
all: $(OBJS)\test.exe $(__test_drawing___depname) $(__test_drawingplugin___depname) $(__test_gui___depname) $(__test_allheaders___depname) data data-image-sample data-images en_GB fr ja
clean:
-if exist $(OBJS)\*.obj del $(OBJS)\*.obj
@ -863,6 +863,10 @@ fr:
if not exist $(OBJS)\intl\fr mkdir $(OBJS)\intl\fr
for %f in (internat.po internat.mo) do if not exist $(OBJS)\intl\fr\%f copy .\intl\fr\%f $(OBJS)\intl\fr
en_GB:
if not exist $(OBJS)\intl\en_GB mkdir $(OBJS)\intl\en_GB
for %f in (internat.po internat.mo) do if not exist $(OBJS)\intl\en_GB\%f copy .\intl\en_GB\%f $(OBJS)\intl\en_GB
ja:
if not exist $(OBJS)\intl\ja mkdir $(OBJS)\intl\ja
for %f in (internat.po internat.mo) do if not exist $(OBJS)\intl\ja\%f copy .\intl\ja\%f $(OBJS)\intl\ja

View file

@ -429,6 +429,7 @@
<files>internat.po internat.mo</files>
</template>
<wx-data id="en_GB" template="catalog"/>
<wx-data id="fr" template="catalog"/>
<wx-data id="ja" template="catalog"/>