# strgen.cpp

Go to the documentation of this file.

    00001 /* $Id: strgen.cpp 26058 2013-11-23 13:15:07Z rubidium $ */
    00002
    00003 /*
    00004  * This file is part of OpenTTD.
    00005  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
    00006  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    00007  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
    00008  */
    00009
    00012 #include "../stdafx.h"
    00013 #include "../core/endian_func.hpp"
    00014 #include "../string_func.h"
    00015 #include "../strings_type.h"
    00016 #include "../misc/getoptdata.h"
    00017 #include "../table/control_codes.h"
    00018
    00019 #include "strgen.h"
    00020
    00021 #include <stdarg.h>
    00022 #include <exception>
    00023
    00024 #if (!defined(WIN32) && !defined(WIN64)) || defined(__CYGWIN__)
    00025 #include <unistd.h>
    00026 #include <sys/stat.h>
    00027 #endif
    00028
    00029 #if defined WIN32 || defined __WATCOMC__
    00030 #include <direct.h>
    00031 #endif /* WIN32 || __WATCOMC__ */
    00032
    00033 #ifdef __MORPHOS__
    00034 #ifdef stderr
    00035 #undef stderr
    00036 #endif
    00037 #define stderr stdout
    00038 #endif /* __MORPHOS__ */
    00039
    00040 #include "../table/strgen_tables.h"
    00041
    00042
    00043 #ifdef _MSC_VER
    00044 # define LINE_NUM_FMT(s) "%s (%d): warning: %s (" s ")\n"
    00045 #else
    00046 # define LINE_NUM_FMT(s) "%s:%d: " s ": %s\n"
    00047 #endif
    00048
    00049 void CDECL strgen_warning(const char *s, ...)
    00050 {
    00051   char buf[1024];
    00052   va_list va;
    00053   va_start(va, s);
    00054   vsnprintf(buf, lengthof(buf), s, va);
    00055   va_end(va);
    00056   fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, buf);
    00057   _warnings++;
    00058 }
    00059
    00060 void CDECL strgen_error(const char *s, ...)
    00061 {
    00062   char buf[1024];
    00063   va_list va;
    00064   va_start(va, s);
    00065   vsnprintf(buf, lengthof(buf), s, va);
    00066   va_end(va);
    00067   fprintf(stderr, LINE_NUM_FMT("error"), _file, _cur_line, buf);
    00068   _errors++;
    00069 }
    00070
    00071 void NORETURN CDECL strgen_fatal(const char *s, ...)
    00072 {
    00073   char buf[1024];
    00074   va_list va;
    00075   va_start(va, s);
    00076   vsnprintf(buf, lengthof(buf), s, va);
    00077   va_end(va);
    00078   fprintf(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, buf);
    00079 #ifdef _MSC_VER
    00080   fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
    00081 #endif
    00082   throw std::exception();
    00083 }
    00084
    00085 void NORETURN CDECL error(const char *s, ...)
    00086 {
    00087   char buf[1024];
    00088   va_list va;
    00089   va_start(va, s);
    00090   vsnprintf(buf, lengthof(buf), s, va);
    00091   va_end(va);
    00092   fprintf(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, buf);
    00093 #ifdef _MSC_VER
    00094   fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
    00095 #endif
    00096   exit(2);
    00097 }
    00098
    00100 struct FileStringReader : StringReader {
    00101   FILE *fh;
    00102
    00110   FileStringReader(StringData &data, const char *file, bool master, bool translation) :
    00111       StringReader(data, file, master, translation)
    00112   {
    00113     this->fh = fopen(file, "rb");
    00114     if (this->fh == NULL) error("Could not open %s", file);
    00115   }
    00116
    00118   virtual ~FileStringReader()
    00119   {
    00120     fclose(this->fh);
    00121   }
    00122
    00123   /* virtual */ char *ReadLine(char *buffer, size_t size)
    00124   {
    00125     return fgets(buffer, size, this->fh);
    00126   }
    00127
    00128   /* virtual */ void HandlePragma(char *str);
    00129
    00130   /* virtual */ void ParseFile()
    00131   {
    00132     this->StringReader::ParseFile();
    00133
    00134     if (StrEmpty(_lang.name) || StrEmpty(_lang.own_name) || StrEmpty(_lang.isocode)) {
    00135       error("Language must include ##name, ##ownname and ##isocode");
    00136     }
    00137   }
    00138 };
    00139
    00140 void FileStringReader::HandlePragma(char *str)
    00141 {
    00142   if (!memcmp(str, "id ", 3)) {
    00143     this->data.next_string_id = strtoul(str + 3, NULL, 0);
    00144   } else if (!memcmp(str, "name ", 5)) {
    00145     strecpy(_lang.name, str + 5, lastof(_lang.name));
    00146   } else if (!memcmp(str, "ownname ", 8)) {
    00147     strecpy(_lang.own_name, str + 8, lastof(_lang.own_name));
    00148   } else if (!memcmp(str, "isocode ", 8)) {
    00149     strecpy(_lang.isocode, str + 8, lastof(_lang.isocode));
    00150   } else if (!memcmp(str, "textdir ", 8)) {
    00151     if (!memcmp(str + 8, "ltr", 3)) {
    00152       _lang.text_dir = TD_LTR;
    00153     } else if (!memcmp(str + 8, "rtl", 3)) {
    00154       _lang.text_dir = TD_RTL;
    00155     } else {
    00156       error("Invalid textdir %s", str + 8);
    00157     }
    00158   } else if (!memcmp(str, "digitsep ", 9)) {
    00159     str += 9;
    00160     strecpy(_lang.digit_group_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_group_separator));
    00161   } else if (!memcmp(str, "digitsepcur ", 12)) {
    00162     str += 12;
    00163     strecpy(_lang.digit_group_separator_currency, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_group_separator_currency));
    00164   } else if (!memcmp(str, "decimalsep ", 11)) {
    00165     str += 11;
    00166     strecpy(_lang.digit_decimal_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_decimal_separator));
    00167   } else if (!memcmp(str, "winlangid ", 10)) {
    00168     const char *buf = str + 10;
    00169     long langid = strtol(buf, NULL, 16);
    00170     if (langid > (long)UINT16_MAX || langid < 0) {
    00171       error("Invalid winlangid %s", buf);
    00172     }
    00173     _lang.winlangid = (uint16)langid;
    00174   } else if (!memcmp(str, "grflangid ", 10)) {
    00175     const char *buf = str + 10;
    00176     long langid = strtol(buf, NULL, 16);
    00177     if (langid >= 0x7F || langid < 0) {
    00178       error("Invalid grflangid %s", buf);
    00179     }
    00180     _lang.newgrflangid = (uint8)langid;
    00181   } else if (!memcmp(str, "gender ", 7)) {
    00182     if (this->master) error("Genders are not allowed in the base translation.");
    00183     char *buf = str + 7;
    00184
    00185     for (;;) {
    00186       const char *s = ParseWord(&buf);
    00187
    00188       if (s == NULL) break;
    00189       if (_lang.num_genders >= MAX_NUM_GENDERS) error("Too many genders, max %d", MAX_NUM_GENDERS);
    00190       strecpy(_lang.genders[_lang.num_genders], s, lastof(_lang.genders[_lang.num_genders]));
    00191       _lang.num_genders++;
    00192     }
    00193   } else if (!memcmp(str, "case ", 5)) {
    00194     if (this->master) error("Cases are not allowed in the base translation.");
    00195     char *buf = str + 5;
    00196
    00197     for (;;) {
    00198       const char *s = ParseWord(&buf);
    00199
    00200       if (s == NULL) break;
    00201       if (_lang.num_cases >= MAX_NUM_CASES) error("Too many cases, max %d", MAX_NUM_CASES);
    00202       strecpy(_lang.cases[_lang.num_cases], s, lastof(_lang.cases[_lang.num_cases]));
    00203       _lang.num_cases++;
    00204     }
    00205   } else {
    00206     StringReader::HandlePragma(str);
    00207   }
    00208 }
    00209
    00210 bool CompareFiles(const char *n1, const char *n2)
    00211 {
    00212   FILE *f2 = fopen(n2, "rb");
    00213   if (f2 == NULL) return false;
    00214
    00215   FILE *f1 = fopen(n1, "rb");
    00216   if (f1 == NULL) error("can't open %s", n1);
    00217
    00218   size_t l1, l2;
    00219   do {
    00220     char b1[4096];
    00221     char b2[4096];
    00222     l1 = fread(b1, 1, sizeof(b1), f1);
    00223     l2 = fread(b2, 1, sizeof(b2), f2);
    00224
    00225     if (l1 != l2 || memcmp(b1, b2, l1)) {
    00226       fclose(f2);
    00227       fclose(f1);
    00228       return false;
    00229     }
    00230   } while (l1 != 0);
    00231
    00232   fclose(f2);
    00233   fclose(f1);
    00234   return true;
    00235 }
    00236
    00238 struct FileWriter {
    00239   FILE *fh;
    00240   const char *filename;
    00241
    00246   FileWriter(const char *filename)
    00247   {
    00248     this->filename = strdup(filename);
    00249     this->fh = fopen(this->filename, "wb");
    00250
    00251     if (this->fh == NULL) {
    00252       error("Could not open %s", this->filename);
    00253     }
    00254   }
    00255
    00257   void Finalise()
    00258   {
    00259     fclose(this->fh);
    00260     this->fh = NULL;
    00261   }
    00262
    00264   virtual ~FileWriter()
    00265   {
    00266     /* If we weren't closed an exception was thrown, so remove the temporary file. */
    00267     if (fh != NULL) {
    00268       fclose(this->fh);
    00269       unlink(this->filename);
    00270     }
    00271     free(this->filename);
    00272   }
    00273 };
    00274
    00275 struct HeaderFileWriter : HeaderWriter, FileWriter {
    00277   const char *real_filename;
    00279   int prev;
    00280
    00285   HeaderFileWriter(const char *filename) : FileWriter("tmp.xxx"),
    00286     real_filename(strdup(filename)), prev(0)
    00287   {
    00288     fprintf(this->fh, "/* This file is automatically generated. Do not modify */\n\n");
    00289     fprintf(this->fh, "#ifndef TABLE_STRINGS_H\n");
    00290     fprintf(this->fh, "#define TABLE_STRINGS_H\n");
    00291   }
    00292
    00294   ~HeaderFileWriter()
    00295   {
    00296     free(real_filename);
    00297   }
    00298
    00299   void WriteStringID(const char *name, int stringid)
    00300   {
    00301     if (prev + 1 != stringid) fprintf(this->fh, "\n");
    00302     fprintf(this->fh, "static const StringID %s = 0x%X;\n", name, stringid);
    00303     prev = stringid;
    00304   }
    00305
    00306   void Finalise(const StringData &data)
    00307   {
    00308     /* Find the plural form with the most amount of cases. */
    00309     int max_plural_forms = 0;
    00310     for (uint i = 0; i < lengthof(_plural_forms); i++) {
    00311       max_plural_forms = max(max_plural_forms, _plural_forms[i].plural_count);
    00312     }
    00313
    00314     fprintf(this->fh,
    00315       "\n"
    00316       "static const uint LANGUAGE_PACK_VERSION     = 0x%X;\n"
    00317       "static const uint LANGUAGE_MAX_PLURAL       = %d;\n"
    00318       "static const uint LANGUAGE_MAX_PLURAL_FORMS = %d;\n\n",
    00319       (uint)data.Version(), (uint)lengthof(_plural_forms), max_plural_forms
    00320     );
    00321
    00322     fprintf(this->fh, "#endif /* TABLE_STRINGS_H */\n");
    00323
    00324     this->FileWriter::Finalise();
    00325
    00326     if (CompareFiles(this->filename, this->real_filename)) {
    00327       /* files are equal. tmp.xxx is not needed */
    00328       unlink(this->filename);
    00329     } else {
    00330       /* else rename tmp.xxx into filename */
    00331   #if defined(WIN32) || defined(WIN64)
    00332       unlink(this->real_filename);
    00333   #endif
    00334       if (rename(this->filename, this->real_filename) == -1) error("rename() failed");
    00335     }
    00336   }
    00337 };
    00338
    00340 struct LanguageFileWriter : LanguageWriter, FileWriter {
    00345   LanguageFileWriter(const char *filename) : FileWriter(filename)
    00346   {
    00347   }
    00348
    00349   void WriteHeader(const LanguagePackHeader *header)
    00350   {
    00351     this->Write((const byte *)header, sizeof(*header));
    00352   }
    00353
    00354   void Finalise()
    00355   {
    00356     if (fputc(0, this->fh) == EOF) {
    00357       error("Could not write to %s", this->filename);
    00358     }
    00359     this->FileWriter::Finalise();
    00360   }
    00361
    00362   void Write(const byte *buffer, size_t length)
    00363   {
    00364     if (fwrite(buffer, sizeof(*buffer), length, this->fh) != length) {
    00365       error("Could not write to %s", this->filename);
    00366     }
    00367   }
    00368 };
    00369
    00371 static inline void ottd_mkdir(const char *directory)
    00372 {
    00373   /* Ignore directory creation errors; they'll surface later on, and most
    00374    * of the time they are 'directory already exists' errors anyhow. */
    00375 #if defined(WIN32) || defined(__WATCOMC__)
    00376   mkdir(directory);
    00377 #else
    00378   mkdir(directory, 0755);
    00379 #endif
    00380 }
    00381
    00387 static inline char *mkpath(char *buf, size_t buflen, const char *path, const char *file)
    00388 {
    00389   ttd_strlcpy(buf, path, buflen); // copy directory into buffer
    00390
    00391   char *p = strchr(buf, '\0'); // add path separator if necessary
    00392   if (p[-1] != PATHSEPCHAR && (size_t)(p - buf) + 1 < buflen) *p++ = PATHSEPCHAR;
    00393   ttd_strlcpy(p, file, buflen - (size_t)(p - buf)); // concatenate filename at end of buffer
    00394   return buf;
    00395 }
    00396
    00397 #if defined(__MINGW32__)
    00398
    00403 static inline char *replace_pathsep(char *s)
    00404 {
    00405   for (char *c = s; *c != '\0'; c++) if (*c == '/') *c = '\\';
    00406   return s;
    00407 }
    00408 #else
    00409 static inline char *replace_pathsep(char *s) { return s; }
    00410 #endif
    00411
    00413 static const OptionData _opts[] = {
    00414     GETOPT_NOVAL(     'v',  "--version"),
    00415   GETOPT_GENERAL('C', '\0', "-export-commands", ODF_NO_VALUE),
    00416   GETOPT_GENERAL('L', '\0', "-export-plurals",  ODF_NO_VALUE),
    00417   GETOPT_GENERAL('P', '\0', "-export-pragmas",  ODF_NO_VALUE),
    00418     GETOPT_NOVAL(     't',  "--todo"),
    00419     GETOPT_NOVAL(     'w',  "--warning"),
    00420     GETOPT_NOVAL(     'h',  "--help"),
    00421   GETOPT_GENERAL('h', '?',  NULL,               ODF_NO_VALUE),
    00422     GETOPT_VALUE(     's',  "--source_dir"),
    00423     GETOPT_VALUE(     'd',  "--dest_dir"),
    00424   GETOPT_END(),
    00425 };
    00426
    00427 int CDECL main(int argc, char *argv[])
    00428 {
    00429   char pathbuf[MAX_PATH];
    00430   const char *src_dir = ".";
    00431   const char *dest_dir = NULL;
    00432
    00433   GetOptData mgo(argc - 1, argv + 1, _opts);
    00434   for (;;) {
    00435     int i = mgo.GetOpt();
    00436     if (i == -1) break;
    00437
    00438     switch (i) {
    00439       case 'v':
    00440         puts("$Revision: 26058 $");
    00441         return 0;
    00442
    00443       case 'C':
    00444         printf("args\tflags\tcommand\treplacement\n");
    00445         for (const CmdStruct *cs = _cmd_structs; cs < endof(_cmd_structs); cs++) {
    00446           char flags;
    00447           if (cs->proc == EmitGender) {
    00448             flags = 'g'; // Command needs number of parameters defined by number of genders
    00449           } else if (cs->proc == EmitPlural) {
    00450             flags = 'p'; // Command needs number of parameters defined by plural value
    00451           } else if (cs->flags & C_DONTCOUNT) {
    00452             flags = 'i'; // Command may be in the translation when it is not in base
    00453           } else {
    00454             flags = '0'; // Command needs no parameters
    00455           }
    00456           printf("%i\t%c\t\"%s\"\t\"%s\"\n", cs->consumes, flags, cs->cmd, strstr(cs->cmd, "STRING") ? "STRING" : cs->cmd);
    00457         }
    00458         return 0;
    00459
    00460       case 'L':
    00461         printf("count\tdescription\tnames\n");
    00462         for (const PluralForm *pf = _plural_forms; pf < endof(_plural_forms); pf++) {
    00463           printf("%i\t\"%s\"\t%s\n", pf->plural_count, pf->description, pf->names);
    00464         }
    00465         return 0;
    00466
    00467       case 'P':
    00468         printf("name\tflags\tdefault\tdescription\n");
    00469         for (size_t i = 0; i < lengthof(_pragmas); i++) {
    00470           printf("\"%s\"\t%s\t\"%s\"\t\"%s\"\n",
    00471               _pragmas[i][0], _pragmas[i][1], _pragmas[i][2], _pragmas[i][3]);
    00472         }
    00473         return 0;
    00474
    00475       case 't':
    00476         _show_todo |= 1;
    00477         break;
    00478
    00479       case 'w':
    00480         _show_todo |= 2;
    00481         break;
    00482
    00483       case 'h':
    00484         puts(
    00485           "strgen - $Revision: 26058 $\n"
    00486           " -v | --version    print version information and exit\n"
    00487           " -t | --todo       replace any untranslated strings with '<TODO>'\n"
    00488           " -w | --warning    print a warning for any untranslated strings\n"
    00489           " -h | -? | --help  print this help message and exit\n"
    00490           " -s | --source_dir search for english.txt in the specified directory\n"
    00491           " -d | --dest_dir   put output file in the specified directory, create if needed\n"
    00492           " -export-commands  export all commands and exit\n"
    00493           " -export-plurals   export all plural forms and exit\n"
    00494           " -export-pragmas   export all pragmas and exit\n"
    00495           " Run without parameters and strgen will search for english.txt and parse it,\n"
    00496           " creating strings.h. Passing an argument, strgen will translate that language\n"
    00497           " file using english.txt as a reference and output <language>.lng."
    00498         );
    00499         return 0;
    00500
    00501       case 's':
    00502         src_dir = replace_pathsep(mgo.opt);
    00503         break;
    00504
    00505       case 'd':
    00506         dest_dir = replace_pathsep(mgo.opt);
    00507         break;
    00508
    00509       case -2:
    00510         fprintf(stderr, "Invalid arguments\n");
    00511         return 0;
    00512     }
    00513   }
    00514
    00515   if (dest_dir == NULL) dest_dir = src_dir; // if dest_dir is not specified, it equals src_dir
    00516
    00517   try {
    00518     /* strgen has two modes of operation. If no (free) arguments are passed
    00519      * strgen generates strings.h to the destination directory. If it is supplied
    00520      * with a (free) parameter the program will translate that language to destination
    00521      * directory. As input english.txt is parsed from the source directory */
    00522     if (mgo.numleft == 0) {
    00523       mkpath(pathbuf, lengthof(pathbuf), src_dir, "english.txt");
    00524
    00525       /* parse master file */
    00526       StringData data(TAB_COUNT);
    00527       FileStringReader master_reader(data, pathbuf, true, false);
    00528       master_reader.ParseFile();
    00529       if (_errors != 0) return 1;
    00530
    00531       /* write strings.h */
    00532       ottd_mkdir(dest_dir);
    00533       mkpath(pathbuf, lengthof(pathbuf), dest_dir, "strings.h");
    00534
    00535       HeaderFileWriter writer(pathbuf);
    00536       writer.WriteHeader(data);
    00537       writer.Finalise(data);
    00538     } else if (mgo.numleft >= 1) {
    00539       char *r;
    00540
    00541       mkpath(pathbuf, lengthof(pathbuf), src_dir, "english.txt");
    00542
    00543       StringData data(TAB_COUNT);
    00544       /* parse master file and check if target file is correct */
    00545       FileStringReader master_reader(data, pathbuf, true, false);
    00546       master_reader.ParseFile();
    00547
    00548       for (int i = 0; i < mgo.numleft; i++) {
    00549         data.FreeTranslation();
    00550
    00551         const char *translation = replace_pathsep(mgo.argv[i]);
    00552         const char *file = strrchr(translation, PATHSEPCHAR);
    00553         FileStringReader translation_reader(data, translation, false, file == NULL || strcmp(file + 1, "english.txt") != 0);
    00554         translation_reader.ParseFile(); // target file
    00555         if (_errors != 0) return 1;
    00556
    00557         /* get the targetfile, strip any directories and append to destination path */
    00558         r = strrchr(mgo.argv[i], PATHSEPCHAR);
    00559         mkpath(pathbuf, lengthof(pathbuf), dest_dir, (r != NULL) ? &r[1] : mgo.argv[i]);
    00560
    00561         /* rename the .txt (input-extension) to .lng */
    00562         r = strrchr(pathbuf, '.');
    00563         if (r == NULL || strcmp(r, ".txt") != 0) r = strchr(pathbuf, '\0');
    00564         ttd_strlcpy(r, ".lng", (size_t)(r - pathbuf));
    00565
    00566         LanguageFileWriter writer(pathbuf);
    00567         writer.WriteLang(data);
    00568         writer.Finalise();
    00569
    00570         /* if showing warnings, print a summary of the language */
    00571         if ((_show_todo & 2) != 0) {
    00572           fprintf(stdout, "%d warnings and %d errors for %s\n", _warnings, _errors, pathbuf);
    00573         }
    00574       }
    00575     }
    00576   } catch (...) {
    00577     return 2;
    00578   }
    00579
    00580   return 0;
    00581 }