Merge pull request #54 from Steffo99/i18n

Closes #51: Better strings and localization support

config/template_config.ini

@@ -5,16 +5,23 @@
 # Config file parameters
 [Config]
 ; Config file version. DO NOT EDIT THIS!
-version = 17
+version = 18
 ; Set this to no when you are done editing the file
 is_template = yes
-; Language code for string file
+
+# Language parameters
+[Language]
 ; Available languages:
-; it_IT - Italian, by Steffo
+; it - Italian, by https://github.com/Steffo99
-; en_US - English, by https://github.com/DarrenWestwood (incomplete, please improve it!)
+; en - English, by https://github.com/DarrenWestwood
-; ua_UK - Ukrainian, by https://github.com/pzhuk
+; uk - Ukrainian, by https://github.com/pzhuk
-; ru_RU - Russian, by https://github.com/pzhuk
+; ru - Russian, by https://github.com/pzhuk
-language = it_IT
+; The lanugages that messages can be displayed in
+enabled_languages = it | en | uk | ru
+; The default language to be set for users whose language cannot be autodetected or whose language is not enabled
+default_language = it
+; The language to fallback to if a string is missing in a specific language
+fallback_language = en
 
 # Telegram bot parameters
 [Telegram]

core.py

@@ -4,7 +4,7 @@ import worker
 import configloader
 import utils
 import threading
-import importlib
+import localization
 import logging
 
 try:
@@ -12,9 +12,6 @@ try:
 except ImportError:
     coloredlogs = None
 
-language = configloader.config["Config"]["language"]
-strings = importlib.import_module("strings." + language)
-
 
 def main():
     """The core code of the program. Should be run only in the main process!"""
@@ -48,6 +45,11 @@ def main():
         sys.exit(1)
     log.debug("Bot token is valid!")
 
+    # Finding default language
+    default_language = configloader.config["Language"]["default_language"]
+    # Creating localization object
+    default_loc = localization.Localization(language=default_language, fallback=default_language)
+
     # Create a dictionary linking the chat ids to the Worker objects
     # {"1234": <Worker>}
     chat_workers = {}
@@ -72,7 +74,7 @@ def main():
                 if update.message.chat.type != "private":
                     log.debug(f"Received a message from a non-private chat: {update.message.chat.id}")
                     # Notify the chat
-                    bot.send_message(update.message.chat.id, strings.error_nonprivate_chat)
+                    bot.send_message(update.message.chat.id, default_loc.get("error_nonprivate_chat"))
                     # Skip the update
                     continue
                 # If the message is a start command...
@@ -85,7 +87,9 @@ def main():
                         log.debug(f"Received request to stop {old_worker.name}")
                         old_worker.stop("request")
                     # Initialize a new worker for the chat
-                    new_worker = worker.Worker(bot, update.message.chat)
+                    new_worker = worker.Worker(bot=bot,
+                                               chat=update.message.chat,
+                                               telegram_user=update.message.from_user)
                     # Start the worker
                     log.debug(f"Starting {new_worker.name}")
                     new_worker.start()
@@ -99,12 +103,12 @@ def main():
                 if receiving_worker is None or not receiving_worker.is_alive():
                     log.debug(f"Received a message in a chat without worker: {update.message.chat.id}")
                     # Suggest that the user restarts the chat with /start
-                    bot.send_message(update.message.chat.id, strings.error_no_worker_for_chat,
+                    bot.send_message(update.message.chat.id, default_loc.get("error_no_worker_for_chat"),
                                      reply_markup=telegram.ReplyKeyboardRemove())
                     # Skip the update
                     continue
                 # If the message contains the "Cancel" string defined in the strings file...
-                if update.message.text == strings.menu_cancel:
+                if update.message.text == receiving_worker.loc.get("menu_cancel"):
                     log.debug(f"Forwarding CancelSignal to {receiving_worker}")
                     # Send a CancelSignal to the worker instead of the update
                     receiving_worker.queue.put(worker.CancelSignal())
@@ -120,7 +124,7 @@ def main():
                 if receiving_worker is None:
                     log.debug(f"Received a callback query in a chat without worker: {update.callback_query.from_user.id}")
                     # Suggest that the user restarts the chat with /start
-                    bot.send_message(update.callback_query.from_user.id, strings.error_no_worker_for_chat)
+                    bot.send_message(update.callback_query.from_user.id, default_loc.get("error_no_worker_for_chat"))
                     # Skip the update
                     continue
                 # Check if the pressed inline key is a cancel button
@@ -146,7 +150,7 @@ def main():
                     try:
                         bot.answer_pre_checkout_query(update.pre_checkout_query.id,
                                                       ok=False,
-                                                      error_message=strings.error_invoice_expired)
+                                                      error_message=default_loc.get("error_invoice_expired"))
                     except telegram.error.BadRequest:
                         log.error("pre-checkout query expired before an answer could be sent!")
                     # Go to the next update

database.py

@@ -7,10 +7,10 @@ import configloader
 import telegram
 import requests
 import utils
-import importlib
+import localization
+import logging
 
-language = configloader.config["Config"]["language"]
+log = logging.getLogger(__name__)
-strings = importlib.import_module("strings." + language)
 
 # Create a (lazy) database engine
 engine = create_engine(configloader.config["Database"]["engine"])
@@ -31,6 +31,7 @@ class User(TableDeclarativeBase):
     first_name = Column(String, nullable=False)
     last_name = Column(String)
     username = Column(String)
+    language = Column(String, nullable=False)
 
     # Current wallet credit
     credit = Column(Integer, nullable=False)
@@ -38,14 +39,16 @@ class User(TableDeclarativeBase):
     # Extra table parameters
     __tablename__ = "users"
 
-    def __init__(self, telegram_chat: telegram.Chat, **kwargs):
+    def __init__(self, telegram_user: telegram.User, **kwargs):
         # Initialize the super
         super().__init__(**kwargs)
         # Get the data from telegram
-        self.user_id = telegram_chat.id
+        self.user_id = telegram_user.id
-        self.first_name = telegram_chat.first_name
+        self.first_name = telegram_user.first_name
-        self.last_name = telegram_chat.last_name
+        self.last_name = telegram_user.last_name
-        self.username = telegram_chat.username
+        self.username = telegram_user.username
+        self.language = telegram_user.language_code if telegram_user.language_code else configloader.config["Language"][
+            "default_language"]
         # The starting wallet value is 0
         self.credit = 0
 
@@ -74,6 +77,13 @@ class User(TableDeclarativeBase):
         valid_transactions: typing.List[Transaction] = [t for t in self.transactions if not t.refunded]
         self.credit = sum(map(lambda t: t.value, valid_transactions))
 
+    @property
+    def full_name(self):
+        if self.last_name:
+            return f"{self.first_name} {self.last_name}"
+        else:
+            return self.first_name
+
     def __repr__(self):
         return f"<User {self} having {self.credit} credit>"
 
@@ -99,40 +109,37 @@ class Product(TableDeclarativeBase):
 
     # No __init__ is needed, the default one is sufficient
 
-    def __str__(self):
+    def text(self, *, loc: localization.Localization, style: str = "full", cart_qty: int = None):
-        return self.text()
-
-    def text(self, style: str="full", cart_qty: int=None):
         """Return the product details formatted with Telegram HTML. The image is omitted."""
         if style == "short":
-            return f"{cart_qty}x {utils.telegram_html_escape(self.name)} - {str(utils.Price(self.price) * cart_qty)}"
+            return f"{cart_qty}x {utils.telegram_html_escape(self.name)} - {str(utils.Price(self.price, loc) * cart_qty)}"
         elif style == "full":
             if cart_qty is not None:
-                cart = strings.in_cart_format_string.format(quantity=cart_qty)
+                cart = loc.get("in_cart_format_string", quantity=cart_qty)
             else:
                 cart = ''
-            return strings.product_format_string.format(name=utils.telegram_html_escape(self.name),
+            return loc.get("product_format_string", name=utils.telegram_html_escape(self.name),
-                                                        description=utils.telegram_html_escape(self.description),
+                           description=utils.telegram_html_escape(self.description),
-                                                        price=str(utils.Price(self.price)),
+                           price=str(utils.Price(self.price, loc)),
-                                                        cart=cart)
+                           cart=cart)
         else:
             raise ValueError("style is not an accepted value")
 
     def __repr__(self):
         return f"<Product {self.name}>"
 
-    def send_as_message(self, chat_id: int) -> dict:
+    def send_as_message(self, loc: localization.Localization, chat_id: int) -> dict:
         """Send a message containing the product data."""
         if self.image is None:
             r = requests.get(f"https://api.telegram.org/bot{configloader.config['Telegram']['token']}/sendMessage",
                              params={"chat_id": chat_id,
-                                     "text": self.text(),
+                                     "text": self.text(loc=loc),
                                      "parse_mode": "HTML"})
         else:
             r = requests.post(f"https://api.telegram.org/bot{configloader.config['Telegram']['token']}/sendPhoto",
                               files={"photo": self.image},
                               params={"chat_id": chat_id,
-                                      "caption": self.text(),
+                                      "caption": self.text(loc=loc),
                                       "parse_mode": "HTML"})
         return r.json()
 
@@ -181,10 +188,10 @@ class Transaction(TableDeclarativeBase):
     __tablename__ = "transactions"
     __table_args__ = (UniqueConstraint("provider", "provider_charge_id"),)
 
-    def __str__(self):
+    def text(self, *, loc: localization.Localization):
-        string = f"<b>T{self.transaction_id}</b> | {str(self.user)} | {utils.Price(self.value)}"
+        string = f"<b>T{self.transaction_id}</b> | {str(self.user)} | {utils.Price(self.value, loc)}"
         if self.refunded:
-            string += f" | {strings.emoji_refunded}"
+            string += f" | {loc.get('emoji_refunded')}"
         if self.provider:
             string += f" | {self.provider}"
         if self.notes:
@@ -247,36 +254,38 @@ class Order(TableDeclarativeBase):
     def __repr__(self):
         return f"<Order {self.order_id} placed by User {self.user_id}>"
 
-    def get_text(self, session, user=False):
+    def text(self, *, loc: localization.Localization, session, user=False):
         joined_self = session.query(Order).filter_by(order_id=self.order_id).join(Transaction).one()
         items = ""
         for item in self.items:
             items += str(item) + "\n"
         if self.delivery_date is not None:
-            status_emoji = strings.emoji_completed
+            status_emoji = loc.get("emoji_completed")
-            status_text = strings.text_completed
+            status_text = loc.get("text_completed")
         elif self.refund_date is not None:
-            status_emoji = strings.emoji_refunded
+            status_emoji = loc.get("emoji_refunded")
-            status_text = strings.text_refunded
+            status_text = loc.get("text_refunded")
         else:
-            status_emoji = strings.emoji_not_processed
+            status_emoji = loc.get("emoji_not_processed")
-            status_text = strings.text_not_processed
+            status_text = loc.get("text_not_processed")
         if user and configloader.config["Appearance"]["full_order_info"] == "no":
-            return strings.user_order_format_string.format(status_emoji=status_emoji,
+            return loc.get("user_order_format_string",
-                                                           status_text=status_text,
+                           status_emoji=status_emoji,
-                                                           items=items,
+                           status_text=status_text,
-                                                           notes=self.notes,
+                           items=items,
-                                                           value=str(utils.Price(-joined_self.transaction.value))) + \
+                           notes=self.notes,
-                (strings.refund_reason.format(reason=self.refund_reason) if self.refund_date is not None else "")
+                           value=str(utils.Price(-joined_self.transaction.value, loc))) + \
+                   (loc.get("refund_reason", reason=self.refund_reason) if self.refund_date is not None else "")
         else:
             return status_emoji + " " + \
-                strings.order_number.format(id=self.order_id) + "\n" + \
+                   loc.get("order_number", id=self.order_id) + "\n" + \
-                strings.order_format_string.format(user=self.user.mention(),
+                   loc.get("order_format_string",
-                                                   date=self.creation_date.isoformat(),
+                           user=self.user.mention(),
-                                                   items=items,
+                           date=self.creation_date.isoformat(),
-                                                   notes=self.notes if self.notes is not None else "",
+                           items=items,
-                                                   value=str(utils.Price(-joined_self.transaction.value))) + \
+                           notes=self.notes if self.notes is not None else "",
-                (strings.refund_reason.format(reason=self.refund_reason) if self.refund_date is not None else "")
+                           value=str(utils.Price(-joined_self.transaction.value, loc))) + \
+                   (loc.get("refund_reason", reason=self.refund_reason) if self.refund_date is not None else "")
 
 
 class OrderItem(TableDeclarativeBase):
@@ -293,8 +302,8 @@ class OrderItem(TableDeclarativeBase):
     # Extra table parameters
     __tablename__ = "orderitems"
 
-    def __str__(self):
+    def text(self, *, loc: localization.Localization):
-        return f"{self.product.name} - {str(utils.Price(self.product.price))}"
+        return f"{self.product.name} - {str(utils.Price(self.product.price, loc))}"
 
     def __repr__(self):
         return f"<OrderItem {self.item_id}>"

localization.py

@@ -0,0 +1,59 @@
+from typing import *
+import importlib
+import types
+import logging
+import json
+
+
+log = logging.getLogger(__name__)
+
+
+class IgnoreDict(dict):
+    """A dictionary that if passed to format_map, ignores the missing replacement fields."""
+    def __missing__(self, key):
+        return "{" + key + "}"
+
+
+class Localization:
+    def __init__(self, language: str, *, fallback: str, replacements: Dict[str, str] = None):
+        log.debug(f"Creating localization for {language}")
+        self.language: str = language
+        log.debug(f"Importing strings.{language}")
+        self.module: types.ModuleType = importlib.import_module(f"strings.{language}")
+        if language != fallback:
+            log.debug(f"Importing strings.{fallback} as fallback")
+            self.fallback_language: str = fallback
+            self.fallback_module = importlib.import_module(f"strings.{fallback}") if fallback else None
+        else:
+            log.debug("Language is the same as the default, not importing any fallback")
+            self.fallback_language = None
+            self.fallback_module = None
+        self.replacements: Dict[str, str] = replacements if replacements else {}
+
+    def get(self, key: str, **kwargs) -> str:
+        try:
+            log.debug(f"Getting localized string with key {key}")
+            string = self.module.__getattribute__(key)
+        except AttributeError:
+            if self.fallback_module:
+                log.warning(f"Missing localized string with key {key}, using default")
+                string = self.fallback_module.__getattribute__(key)
+            else:
+                raise
+        assert isinstance(string, str)
+        formatter = IgnoreDict(**self.replacements, **kwargs)
+        return string.format_map(formatter)
+
+    def boolmoji(self, boolean: bool) -> str:
+        return self.get("emoji_yes") if boolean else self.get("emoji_no")
+
+
+def create_json_localization_file_from_strings(language: str):
+    module: types.ModuleType = importlib.import_module(f"strings.{language}")
+    raw = module.__dict__
+    clean = {}
+    for key in raw:
+        if not (key.startswith("__") and key.endswith("__")):
+            clean[key] = raw[key]
+    with open(f"locale/{language}.json", "w") as file:
+        json.dump(clean, file)

strings/en.py

@@ -118,6 +118,9 @@ conversation_open_help_menu = "What kind of help do you need?"
 conversation_confirm_admin_promotion = "Are you sure you want to promote this user to 💼 Manager?\n" \
                                        "It is an irreversible action!"
 
+# Conversation: language select menu header
+conversation_language_select = "Select a language:"
+
 # Conversation: switching to user mode
 conversation_switch_to_user_mode = " You are switching to 👤 Customer mode.\n" \
                                    "If you want to go back to the 💼 Manager menu, restart the conversation with /start."
@@ -214,6 +217,9 @@ menu_csv = "📄 .csv"
 # Menu: edit admins list
 menu_edit_admins = "🏵 Edit Managers"
 
+# Menu: language
+menu_language = "🇬🇧 Language"
+
 # Emoji: unprocessed order
 emoji_not_processed = "*️⃣"
 

strings/it.py

@@ -118,6 +118,9 @@ conversation_open_help_menu = "Che tipo di assistenza desideri ricevere?"
 conversation_confirm_admin_promotion = "Sei sicuro di voler promuovere questo utente a 💼 Gestore?\n" \
                                        "E' un'azione irreversibile!"
 
+# Conversation: language select menu header
+conversation_language_select = "Scegli una lingua:"
+
 # Conversation: switching to user mode
 conversation_switch_to_user_mode = "Stai passando alla modalità 👤 Cliente.\n" \
                                    "Se vuoi riassumere il ruolo di 💼 Gestore, riavvia la conversazione con /start."
@@ -214,6 +217,9 @@ menu_csv = "📄 .csv"
 # Menu: edit admins list
 menu_edit_admins = "🏵 Modifica gestori"
 
+# Menu: language
+menu_language = "🇮🇹 Lingua"
+
 # Emoji: unprocessed order
 emoji_not_processed = "*️⃣"
 

utils.py

@@ -8,18 +8,12 @@ import sys
 import importlib
 import logging
 import traceback
+import localization
 
 
 log = logging.getLogger(__name__)
 
 
-language = config["Config"]["language"]
-try:
-    strings = importlib.import_module("strings." + language)
-except ModuleNotFoundError:
-    print("The strings file you specified in the config file does not exist.")
-    sys.exit(1)
-
 if config["Error Reporting"]["sentry_token"] != \
         "https://00000000000000000000000000000000:00000000000000000000000000000000@sentry.io/0000000":
     import raven
@@ -39,7 +33,9 @@ class Price:
     """The base class for the prices in greed.
     Its int value is in minimum units, while its float and str values are in decimal format.int("""
 
-    def __init__(self, value: typing.Union[int, float, str, "Price"] = 0):
+    def __init__(self, value: typing.Union[int, float, str, "Price"], loc: localization.Localization):
+        # Keep a reference to the localization file
+        self.loc = loc
         if isinstance(value, int):
             # Keep the value as it is
             self.value = int(value)
@@ -57,9 +53,9 @@ class Price:
         return f"<Price of value {self.value}>"
 
     def __str__(self):
-        return strings.currency_format_string.format(symbol=(config["Payments"]["currency_symbol"] or strings.currency_symbol),
+        return self.loc.get("currency_format_string",
-                                                     value="{0:.2f}".format(
+                            symbol=config["Payments"]["currency_symbol"],
-                                                         self.value / (10 ** int(config["Payments"]["currency_exp"]))))
+                            value="{0:.2f}".format(self.value / (10 ** int(config["Payments"]["currency_exp"]))))
 
     def __int__(self):
         return self.value
@@ -68,48 +64,48 @@ class Price:
         return self.value / (10 ** int(config["Payments"]["currency_exp"]))
 
     def __ge__(self, other):
-        return self.value >= Price(other).value
+        return self.value >= Price(other, self.loc).value
 
     def __le__(self, other):
-        return self.value <= Price(other).value
+        return self.value <= Price(other, self.loc).value
 
     def __eq__(self, other):
-        return self.value == Price(other).value
+        return self.value == Price(other, self.loc).value
 
     def __gt__(self, other):
-        return self.value > Price(other).value
+        return self.value > Price(other, self.loc).value
 
     def __lt__(self, other):
-        return self.value < Price(other).value
+        return self.value < Price(other, self.loc).value
 
     def __add__(self, other):
-        return Price(self.value + Price(other).value)
+        return Price(self.value + Price(other, self.loc).value, self.loc)
 
     def __sub__(self, other):
-        return Price(self.value - Price(other).value)
+        return Price(self.value - Price(other, self.loc).value, self.loc)
 
     def __mul__(self, other):
-        return Price(int(self.value * other))
+        return Price(int(self.value * other), self.loc)
 
     def __floordiv__(self, other):
-        return Price(int(self.value // other))
+        return Price(int(self.value // other), self.loc)
 
     def __radd__(self, other):
         return self.__add__(other)
 
     def __rsub__(self, other):
-        return Price(Price(other).value - self.value)
+        return Price(Price(other, self.loc).value - self.value, self.loc)
 
     def __rmul__(self, other):
 
         return self.__mul__(other)
 
     def __iadd__(self, other):
-        self.value += Price(other).value
+        self.value += Price(other, self.loc).value
         return self
 
     def __isub__(self, other):
-        self.value -= Price(other).value
+        self.value -= Price(other, self.loc).value
         return self
 
     def __imul__(self, other):
@@ -235,7 +231,3 @@ class DuckBot:
         return self.bot.send_document(*args, **kwargs)
 
     # More methods can be added here
-
-
-def boolmoji(boolean: bool):
-    return strings.emoji_yes if boolean else strings.emoji_no