update app.py

README.md

@@ -1,13 +1,14 @@
 ---
-title: GlotLID Space
+title: Language identification comparison
 emoji: 📐
-colorFrom: yellow
-colorTo: red
+colorFrom: indigo
+colorTo: blue
 sdk: streamlit
 sdk_version: 1.27.2
 app_file: app.py
 pinned: true
-tags: [multilingual]
+tags:
+- multilingual
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -69,7 +69,7 @@ def preprocess_text(text):
     replace_by = " " 
     replacement_map = {
         ord(c): replace_by
-        for c in ':#{|}' + string.digits
+        for c in ':•#{|}' + string.digits
     }
     text = text.translate(replacement_map)
 
@@ -141,20 +141,18 @@ def convert_df(df):
 
 
 @st.cache_resource
-def load_GlotLID_v1(model_name, file_name):
+def load_model(model_name, file_name):
     model_path = hf_hub_download(repo_id=model_name, filename=file_name)
     model = fasttext.load_model(model_path)
     return model
 
-@st.cache_resource
-def load_GlotLID_v2(model_name, file_name):
-    model_path = hf_hub_download(repo_id=model_name, filename=file_name)
-    model = fasttext.load_model(model_path)
-    return model
 
+model_1 = load_model(constants.MODEL_NAME, "model_v1.bin")
+model_2 = load_model(constants.MODEL_NAME, "model_v2.bin")
+model_3 = load_model(constants.MODEL_NAME, "model_v3.bin")
+openlid = load_model('laurievb/OpenLID', "model.bin")
+nllb = load_model('facebook/fasttext-language-identification', "model.bin")
 
-model_1 = load_GlotLID_v1(constants.MODEL_NAME, "model_v1.bin")
-model_2 = load_GlotLID_v2(constants.MODEL_NAME, "model_v2.bin")
 
 # @st.cache_resource
 def plot(label, prob):
@@ -179,7 +177,7 @@ def plot(label, prob):
     ax.set_xlabel("Confidence", color=BLACK_COLOR)
     st.pyplot(fig)
 
-def compute(sentences, version = 'v2'):
+def compute(sentences, version = 'v3'):
     """Computes the language probablities and labels for the given sentences.
 
     Args:
@@ -189,7 +187,18 @@ def compute(sentences, version = 'v2'):
         A list of language probablities and labels for the given sentences.
     """
     progress_text = "Computing Language..."
-    model_choice = model_2 if version == 'v2' else model_1
+
+    if version == 'nllb-218':
+        model_choice = nllb
+    elif version == 'openlid-201':
+        model_choice = openlid
+    elif version == 'v3':
+        model_choice = model_3   
+    elif version == 'v2':
+        model_choice = model_2
+    else:
+        model_choice = model_1
+        
     my_bar = st.progress(0, text=progress_text)
 
     probs = []
@@ -201,12 +210,12 @@ def compute(sentences, version = 'v2'):
 
         output = model_choice.predict(sent)
         
-        output_label  = output[0][0].split('__')[-1]
+        output_label  = output[0][0].split('__')[-1].replace('_Hans', '_Hani').replace('_Hant', '_Hani')
         output_prob = max(min(output[1][0], 1), 0) 
         output_label_language = output_label.split('_')[0]
 
         # script control
-        if version in ['v2'] and output_label_language!= 'zxx':
+        if version in ['v3', 'v2', 'openlid-201', 'nllb-218'] and output_label_language!= 'zxx':
             main_script, all_scripts = get_script(sent)
             output_label_script = output_label.split('_')[1]
 
@@ -226,13 +235,14 @@ def compute(sentences, version = 'v2'):
     my_bar.empty()
     return probs, labels
 
-st.markdown("[![Duplicate Space](https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14)](https://huggingface.co/spaces/cis-lmu/glotlid-space?duplicate=true)")
+#  st.markdown("[![Duplicate Space](https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14)](https://huggingface.co/spaces/cis-lmu/glotlid-space?duplicate=true)")
 
-render_svg(open("assets/glotlid_logo.svg").read())
+#  render_svg(open("assets/glotlid_logo.svg").read())
 
 render_metadata()
 
-st.markdown("**GlotLID** is an open-source language identification model with support for more than **1600 languages**.")
+st.markdown("**GlotLID** is an open-source language identification model with support for more than **2000 languages (V3)**.")
+
 
 
 tab1, tab2 = st.tabs(["Input a Sentence", "Upload a File"])
@@ -247,9 +257,9 @@ with tab1:
 
     version = st.radio(
         "Choose model",
-        ["v1", "v2"],
-        captions=["GlotLID version 1", "GlotLID version 2 (more data and languages)"],
-        index = 1,
+        ["nllb-218", "openlid-201", "v1", "v2", "v3"],
+        captions=["NLLB", "OpenLID", "GlotLID version 1", "GlotLID version 2", "GlotLID version 3 (More languages, better quality data)"],
+        index = 4,
         key = 'version_tab1',
         horizontal = True
     )
@@ -286,9 +296,9 @@ with tab2:
 
     version = st.radio(
         "Choose model",
-        ["v1", "v2"],
-        captions=["GlotLID version 1", "GlotLID version 2 (more data and languages)"],
-        index = 1,
+        ["nllb-218", "openlid-201", "v1", "v2", "v3"],
+        captions=["NLLB", "OpenLID", "GlotLID version 1", "GlotLID version 2 (more data and languages), GlotLID version 3 (More languages, better quality data)"],
+        index = 4,
         key = 'version_tab2',
         horizontal = True
     )