Add multilanguage support ;-)

Add prompt for english and german

app.py

@@ -4,9 +4,9 @@ from mtranslate import translate
 from prompts import PROMPT_LIST
 import streamlit as st
 import random
+import fasttext
 
-token = st.secrets["flax_community_token"]
-headers = {"Authorization": f"Bearer {token}"}
+headers = {}
 LOGO = "huggingwayang.png"
 MODELS = {
     "GPT-2 Small": {
@@ -48,9 +48,15 @@ def process(text: str,
     return query(payload, model_name)
 
 st.set_page_config(page_title="Indonesian GPT-2 Demo")
-
 st.title("Indonesian GPT-2")
 
+try:
+    token = st.secrets["flax_community_token"]
+    headers = {"Authorization": f"Bearer {token}"}
+except FileNotFoundError:
+    print(f"Token is not found")
+
+ft_model = fasttext.load_model('lid.176.ftz')
 # Sidebar
 st.sidebar.image(LOGO)
 st.sidebar.subheader("Configurable parameters")
@@ -81,7 +87,7 @@ top_p = st.sidebar.number_input(
     help=" If set to float < 1, only the most probable tokens with probabilities that add up to top_p or higher are kept for generation."
 )
 
-do_sample = st.sidebar.selectbox('Sampling?', (True, False), help="Whether or not to use sampling; use greedy decoding otherwise.")
+# do_sample = st.sidebar.selectbox('Sampling?', (True, False), help="Whether or not to use sampling; use greedy decoding otherwise.")
 
 st.markdown(
     """
@@ -90,6 +96,7 @@ st.markdown(
     trained on the Indonesian [Oscar](https://huggingface.co/datasets/oscar), [MC4](https://huggingface.co/datasets/mc4) 
     and [Wikipedia](https://huggingface.co/datasets/wikipedia) dataset. We created it as part of the 
     [Huggingface JAX/Flax event](https://discuss.huggingface.co/t/open-to-the-community-community-week-using-jax-flax-for-nlp-cv/).
+    It is also possible to write a custom prompt on your own language.
     """
 )
 
@@ -99,7 +106,7 @@ ALL_PROMPTS = list(PROMPT_LIST.keys())+["Custom"]
 prompt = st.selectbox('Please choose a predefined prompt or create your custom text.', ALL_PROMPTS, index=len(ALL_PROMPTS)-1)
 
 if prompt == "Custom":
-    prompt_box = "Enter your text here"
+    prompt_box = "Feel free to write text in any language"
 else:
     prompt_box = random.choice(PROMPT_LIST[prompt])
 
@@ -107,6 +114,14 @@ text = st.text_area("Enter text", prompt_box)
 
 if st.button("Run"):
     with st.spinner(text="Getting results..."):
+        lang_predictions, lang_probability = ft_model.predict(text, k=3)
+        # print(f"lang: {lang_predictions}, {lang_probability}")
+        if "__label__id" in lang_predictions:
+            lang = "id"
+        else:
+            lang = lang_predictions[0].replace("__label__", "")
+            text = translate(text, "id", lang)
+        # print(f"{lang}: {text}")
         st.subheader("Result")
         # print(f"maxlen:{max_len}, temp:{temp}, top_k:{top_k}, top_p:{top_p}")
         result = process(text=text,
@@ -129,5 +144,8 @@ if st.button("Run"):
         else:
             result = result[0]["generated_text"]
             st.write(result.replace("\n", "  \n"))
-            st.text("English translation")
-            st.write(translate(result, "en", "id").replace("\n", "  \n"))
+            st.text("Translation")
+            if lang == "id":
+                st.write(translate(result, "en", "id").replace("\n", "  \n"))
+            else:
+                st.write(translate(result, lang, "id").replace("\n", "  \n"))

prompts.py

@@ -17,5 +17,13 @@ PROMPT_LIST = {
         "Mohammad Natsir adalah seorang ulama, politisi, dan pejuang kemerdekaan Indonesia.",
         "Ir. H. Soekarno adalah Presiden pertama Republik Indonesia. Ia adalah seorang tokoh perjuangan yang memainkan peranan penting dalam memerdekakan bangsa Indonesia",
         "Borobudur adalah sebuah candi Buddha yang terletak di sebelah barat laut Yogyakarta. Monumen ini merupakan model alam semesta dan dibangun sebagai tempat suci untuk memuliakan Buddha"
+    ],
+    "English": [
+        "Deoxyribonucleic acid is a molecule composed of two polynucleotide chains that coil around each other",
+        "Javanese is the largest of the Austronesian languages in number of native speakers"
+    ],
+    "German": [
+        "Eine Meerjungfrau, auch Seejungfrau oder Fischweib, ist ein weibliches Fabelwesen, ein Mischwesen aus Frauen- und Fischkörper",
+        "Der Mond ist der einzige natürliche Satellit der Erde"
     ]
 }