Testing: improved gettext files generator to avoir syntax errors

tests/scripts/lang/generate_gettext_files.php

@@ -34,13 +34,15 @@ foreach ($languagesListFull as $language) {
  * Get English language terms (the main source of terms)
  */
 $path = $langPath.'english';
-$terms = [];
+$terms = $originalTerms = [];
 $file = $path.'/trad4all.inc.php';
 if (is_file($file)) {
-    $terms = SubLanguageManager::get_all_language_variable_in_file($file, true);
+    $originalTerms = SubLanguageManager::get_all_language_variable_in_file($file, true);
 }
-foreach ($terms as $index => $translation) {
-    $terms[$index] = trim(rtrim($translation, ';'), '"');
+foreach ($originalTerms as $index => $translation) {
+    if (!isset($terms[$index])) {
+        $terms[$index] = trim(rtrim($translation, ';'), '"');
+    }
 }
 // get only the array keys (the language variables defined in language files)
 //$definedTerms = array_flip(array_keys($terms));
@@ -48,42 +50,42 @@ echo count($terms)." terms were found in language files".PHP_EOL;
 
 // make sure we have an ISO 639-1 (or-2 if no -1) to give the gettext file
 $langToIso639v1 = [
-    'arabic'  => 'ar_AR',
+    'arabic'  => 'ar',
     'asturian' => 'ast_ES',
     'basque'  => 'eu_ES',
     'bengali' => 'bn_BD',
     'bosnian' => 'bs_BA',
     'brazilian' => 'pt_BR',
     'bulgarian' => 'bg_BG',
-    'catalan' => 'ca_ES',
+    'catalan' => 'ca',
     'croatian' => 'hr_HR',
     'czech'   => 'cs_CZ',
-    'danish'  => 'da_DK',
+    'danish'  => 'da',
     'dari'    => 'prs_AF',
-    'dutch'   => 'nl_NL',
+    'dutch'   => 'nl',
     'english' => 'en_US',
     'esperanto' => 'eo',
     'faroese' => 'fo_FO',
     'finnish' => 'fi_FI',
     'french'  => 'fr_FR',
-    'friulian' => 'fur_IT',
-    'galician' => 'gl_ES',
+    'friulian' => 'fur',
+    'galician' => 'gl',
     'georgian' => 'ka_GE',
-    'german'  => 'de_DE',
-    'greek'   => 'el_GR',
+    'german'  => 'de',
+    'greek'   => 'el',
     'hebrew'  => 'he_IL',
     'hindi'   => 'hi_IN',
     'hungarian' => 'hu_HU',
     'indonesian' => 'id_ID',
-    'italian' => 'it_IT',
-    'japanese' => 'ja_JP',
+    'italian' => 'it',
+    'japanese' => 'ja',
     'korean'  => 'ko_KR',
     'latvian' => 'lv_LV',
     'lithuanian' => 'lt_LT',
     'macedonian' => 'mk_MK',
     'malay' => 'ms_MY',
     'norwegian' => 'nn_NO',
-    'occitan' => 'oc_FR',
+    'occitan' => 'oc',
     'pashto' => 'ps_AF',
     'persian' => 'fa_IR',
     'polish' => 'pl_PL',
@@ -96,15 +98,15 @@ $langToIso639v1 = [
     'slovak' => 'sk_SK',
     'slovenian' => 'sl_SI',
     'somali' => 'so_SO',
-    'spanish' => 'es_ES',
+    'spanish' => 'es',
     'spanish_latin' => 'es_MX',
     'swahili' => 'sw_KE',
     'swedish' => 'sv_SE',
     'tagalog' => 'tl_PH',
-    'thai' => 'th_TH',
+    'thai' => 'th',
     'tibetan' => 'bo_CN',
     'trad_chinese' => 'zh_TW',
-    'turkish' => 'tr_TR',
+    'turkish' => 'tr',
     'ukrainian' => 'uk_UA',
     'vietnamese' => 'vi_VN',
     'xhosa' => 'xh_ZA',
@@ -190,7 +192,7 @@ $destinationDir .= '/';
 $destinationFilePot = $destinationDir.'/'.$baseFilename.'/'.$baseFilename.'.pot';
 
 foreach ($languagesList as $language) {
-    $termsInLanguage = [];
+    $termsInLanguage = $originalTermsInLanguage = [];
     $file = $langPath.$language.'/trad4all.inc.php';
     $languageCode = $langToIso639v1[$language];
     $languageFilename = $langToPOFilename[$language];
@@ -202,17 +204,24 @@ foreach ($languagesList as $language) {
         '"Content-Transfer-Encoding: 8bit\n"'."\n\n";
     file_put_contents($destinationFile, $header);
 
-    $termsInLanguage = SubLanguageManager::get_all_language_variable_in_file(
+    $originalTermsInLanguage = SubLanguageManager::get_all_language_variable_in_file(
         $file,
         true
     );
-    foreach ($termsInLanguage as $id => $content) {
-        $termsInLanguage[$id] = trim(rtrim($content, ';'), '"');
+    foreach ($originalTermsInLanguage as $id => $content) {
+        if (!isset($termsInLanguage[$id])) {
+            $termsInLanguage[$id] = trim(rtrim($content, ';'), '"');
+        }
     }
 
     $bigString = '';
     $bigStringPot = '';
+    $doneTranslations = [];
     foreach ($terms as $term => $englishTranslation) {
+        if (isset($doneTranslations[$englishTranslation])) {
+            continue;
+        }
+        $doneTranslations[$englishTranslation] = true;
         $translatedTerm = '';
         if (!empty($termsInLanguage[$term])) {
             $translatedTerm = $termsInLanguage[$term];
@@ -222,6 +231,20 @@ foreach ($languagesList as $language) {
                 continue;
             }
         }
+        // Here we apply a little correction to avoid unterminated strings
+        // when a string ends with a \"
+        if (preg_match('/\\\$/', $englishTranslation)) {
+            $englishTranslation .= '"';
+        }
+        $englishTranslation2 = '';
+        $search = ['\\{', '\\}', '\\(', '\\)', '\\;'];
+        $replace = ['\\\\{', '\\\\}', '\\\\(', '\\\\)', '\\\\;'];
+        $englishTranslation = str_replace($search, $replace, $englishTranslation);
+        if (preg_match('/\\\$/', $translatedTerm)) {
+            $translatedTerm .= '"';
+        }
+        $translatedTerm = str_replace($search, $replace, $translatedTerm);
+        // Now build the line
         $bigString .= 'msgid "'.$englishTranslation.'"'."\n".'msgstr "'.$translatedTerm.'"'."\n\n";
         if ($language === 'english') {
             //$bigStringPot .= 'msgid "'.$term.'"'."\n".'msgstr ""'."\n\n";