merge

app_2.py

@@ -0,0 +1,378 @@
+# Run with: streamlit run visualization.py
+
+import streamlit as st
+
+import os
+
+import base64
+import json
+import pandas as pd
+
+import numpy as np
+
+import matplotlib.pyplot as plt
+
+
+class Visualization:
+    def __init__(
+        self,
+        path_instructions,
+        path_data,
+        lang,
+        num_docs,
+        num_docs_for_words,
+        max_len_text_display,
+    ):
+        self.path_instructions = path_instructions
+        self.path_data = path_data
+        self.lang = lang
+        self.num_docs = num_docs
+        self.num_docs_for_words = num_docs_for_words
+        self.max_len_text_display = max_len_text_display
+
+    def preamble(self):
+        st.markdown(
+            "Before diving into this demo, you might want to take a look at how the filtering pipeline of OSCAR looks like in more detail."
+        )
+
+        def get_binary_file_downloader_html(bin_file, file_label="File"):
+            with open(bin_file, "rb") as f:
+                data = f.read()
+            bin_str = base64.b64encode(data).decode()
+            href = f'<a href="data:application/octet-stream;base64,{bin_str}" download="{os.path.basename(bin_file)}">{file_label}</a>'
+            return href
+
+        st.markdown(
+            get_binary_file_downloader_html(
+                self.path_instructions,
+                "Download the filtering pipeline of OSCAR as pdf",
+            ),
+            unsafe_allow_html=True,
+        )
+
+    def open_data(self):
+        with open(self.path_data) as json_file:
+            data = json.load(json_file)
+
+        self.num_docs = min(self.num_docs, len(data))
+        self.num_docs_for_words = min(self.num_docs_for_words, len(data))
+
+        if "words" in data[0]:
+            words = [doc["words"] for doc in data[: self.num_docs_for_words]]
+            words = [word for doc in words for word in doc]
+            self.words = pd.DataFrame(words)
+        else:
+            self.words = None
+
+        docs = data[: self.num_docs]
+        for doc in docs:
+            if not (self.words is None):
+                del doc["words"]
+            if len(doc["text"]) > self.max_len_text_display:
+                doc["text"] = (
+                    doc["text"][: self.max_len_text_display]
+                    + " [...] [THIS LONG TEXT HAS BEEN TRUNCATED FOR DISPLAY REASONS]"
+                )
+        self.docs = pd.DataFrame(docs)
+
+    def set_title(self):
+        st.title(f"{self.num_docs} {self.lang} documents from OSCAR with their stats.")
+
+    def filtering_of_docs(self):
+        st.sidebar.subheader("Parameters of the filtering on documents")
+
+        def set_sliders(docs):
+            columns = list(docs)
+            keys = []
+            conds = {}
+
+            def get_cond(key, cutoff, max_cutoff):
+                if max_cutoff:
+                    return self.docs[key] <= cutoff
+                return self.docs[key] >= cutoff
+
+            def print_discared_by_cond(cond):
+                st.sidebar.caption(
+                    f"{(len(cond) - np.sum(1*cond)) / len(cond) * 100:.2f}% of the total is discarded with this filter."
+                )
+                st.sidebar.caption("---------")
+
+            if "number_words" in columns:
+                cutoff_def = "If the number of words of a document is lower than this number, the document is removed."
+                max_nb_words = int(np.max(docs["number_words"])) + 1
+                cutoff_min_number_words = st.sidebar.slider(
+                    cutoff_def, 0, min(max_nb_words, 500), 0
+                )
+                new_key = ("number_words", cutoff_min_number_words, False)
+                keys.append(new_key)
+                cond_1 = get_cond(new_key[0], new_key[1], new_key[2])
+                print_discared_by_cond(cond_1)
+
+                cutoff_def = "If the number of words of a document is higher than this number, the document is removed."
+                cutoff_max_number_words = st.sidebar.slider(
+                    cutoff_def, 0, max_nb_words, max_nb_words
+                )
+                new_key = ("number_words", cutoff_max_number_words, True)
+                keys.append(new_key)
+                cond_2 = get_cond(new_key[0], new_key[1], new_key[2])
+                print_discared_by_cond(cond_2)
+
+                conds["number_words"] = [cond_1, cond_2]
+
+            if "special_characters_ratio" in columns:
+                cutoff_def = "If the special characters ratio of a document is higher than this number, the document is removed."
+                cutoff_special_characters_ratio = st.sidebar.slider(
+                    cutoff_def, 0.0, 1.0, 1.0, step=0.01
+                )
+                new_key = (
+                    "special_characters_ratio",
+                    cutoff_special_characters_ratio,
+                    True,
+                )
+                keys.append(new_key)
+                cond = get_cond(new_key[0], new_key[1], new_key[2])
+                print_discared_by_cond(cond)
+                conds["special_characters_ratio"] = [cond]
+
+            if "stopwords_ratio" in columns:
+                cutoff_def = "If the stop words ratio of a document is lower than this number, the document is removed."
+                cutoff_stopwords_ratio = st.sidebar.slider(
+                    cutoff_def, 0.0, 1.0, 0.0, step=0.01
+                )
+                new_key = ("stopwords_ratio", cutoff_stopwords_ratio, False)
+                keys.append(new_key)
+                cond = get_cond(new_key[0], new_key[1], new_key[2])
+                print_discared_by_cond(cond)
+                conds["stopwords_ratio"] = [cond]
+
+            if "badwords_ratio" in columns:
+                cutoff_def = "If the bad words ratio of a document is higher than this number, the document is removed."
+                cutoff_badwords_ratio = st.sidebar.slider(
+                    cutoff_def, 0.0, 1.0, 1.0, step=0.01
+                )
+                new_key = ("badwords_ratio", cutoff_badwords_ratio, True)
+                keys.append(new_key)
+                cond = get_cond(new_key[0], new_key[1], new_key[2])
+                print_discared_by_cond(cond)
+                conds["badwords_ratio"] = [cond]
+
+            if "lang_id_score" in columns:
+                cutoff_def = "If the confidence score for the language identification prediction of a document is lower than this number, the document is removed."
+                cutoff_lang_id_score = st.sidebar.slider(
+                    cutoff_def, 0.0, 1.0, 0.0, step=0.01
+                )
+                new_key = ("lang_id_score", cutoff_lang_id_score, False)
+                keys.append(new_key)
+                cond = get_cond(new_key[0], new_key[1], new_key[2])
+                print_discared_by_cond(cond)
+                conds["lang_id_score"] = [cond]
+
+            if "perplexity_score" in columns:
+                cutoff_def = "If the perplexity score of a document is higher than this number, the document is removed."
+                max_pp = int(np.max(docs["perplexity_score"])) + 1
+                cutoff_perplexity_score = st.sidebar.slider(
+                    cutoff_def, 0, max_pp, max_pp
+                )
+                new_key = ("perplexity_score", cutoff_perplexity_score, True)
+                keys.append(new_key)
+                cond = get_cond(new_key[0], new_key[1], new_key[2])
+                print_discared_by_cond(cond)
+                conds["perplexity_score"] = [cond]
+
+            return keys, conds
+
+        self.keys, conds = set_sliders(self.docs)
+
+        all_conds = [subcond for cond in list(conds.values()) for subcond in cond]
+        all_conds = np.all(all_conds, axis=0)
+
+        st.header("Filtering on documents")
+
+        def display_dataset(cond, description):
+            displayed_docs = self.docs.loc[cond]
+            st.subheader(
+                f"{description}: {len(displayed_docs)} docs ({len(displayed_docs) / self.num_docs * 100:.2f}%)"
+            )
+            st.markdown(
+                "Click on a column to sort by it, place the cursor on the text to display it."
+            )
+            st.dataframe(displayed_docs)
+
+        display_dataset(np.invert(all_conds), "Discarded documents")
+
+        # st.subheader("Display discarded documents by filter")
+        display_discarded_documents_by_filter = st.checkbox(
+            "Display discarded documents by filter"
+        )
+
+        if display_discarded_documents_by_filter:
+            columns = list(self.docs)
+
+            if "number_words" in columns:
+                cond_filter = np.invert(np.all(conds["number_words"], axis=0))
+                display_dataset(
+                    cond_filter,
+                    "Discarded documents for the filter on the number of words",
+                )
+
+            if "special_characters_ratio" in columns:
+                cond_filter = np.invert(
+                    np.all(conds["special_characters_ratio"], axis=0)
+                )
+                display_dataset(
+                    cond_filter,
+                    "Discarded documents for the filter on the special characters ratio",
+                )
+
+            if "stopwords_ratio" in columns:
+                cond_filter = np.invert(np.all(conds["stopwords_ratio"], axis=0))
+                display_dataset(
+                    cond_filter,
+                    "Discarded documents for the filter on the stop words ratio",
+                )
+
+            if "badwords_ratio" in columns:
+                cond_filter = np.invert(np.all(conds["badwords_ratio"], axis=0))
+                display_dataset(
+                    cond_filter,
+                    "Discarded documents for the filter on the bad words ratio",
+                )
+
+            if "lang_id_score" in columns:
+                cond_filter = np.invert(np.all(conds["lang_id_score"], axis=0))
+                display_dataset(
+                    cond_filter,
+                    "Discarded documents for the filter on the language identification confidence score",
+                )
+
+            if "perplexity_score" in columns:
+                cond_filter = np.invert(np.all(conds["perplexity_score"], axis=0))
+                display_dataset(
+                    cond_filter,
+                    "Discarded documents for the filter on the perplexity score",
+                )
+
+        display_dataset(all_conds, "Retained documents")
+
+    def filtering_of_words(self):
+        if not (self.words is None):
+            st.sidebar.subheader("Parameter of the filtering on words")
+
+            cutoff_def = "If the length of a word is higher than this number, the word is removed."
+            max_len_word = min(int(np.max(self.words["len_word"])) + 1, 200)
+            cutoff_word = st.sidebar.slider(cutoff_def, 0, max_len_word, max_len_word)
+
+            incorrect_substrings = st.sidebar.checkbox(
+                "Remove words with incorrect substrings."
+            )
+
+            cond_words = self.words["len_word"] <= cutoff_word
+            if incorrect_substrings:
+                cond_words = cond_words & np.invert(self.words["incorrect_substring"])
+
+            st.header("Filtering on words")
+
+            st.markdown(
+                f"Since the number of words is way larger than the number of documents, "
+                f"we consider in this section words for the first {self.num_docs_for_words} documents only."
+            )
+
+            discarded_words = self.words.loc[np.invert(cond_words)]
+            st.subheader(
+                f"Discarded words: {len(discarded_words)} words ({len(discarded_words) / len(self.words) * 100:.2f}%)"
+            )
+            st.markdown(
+                "Click on a column to sort by it, place the cursor on the text to display it."
+            )
+            st.dataframe(discarded_words)
+
+            retained_words = self.words.loc[cond_words]
+            st.subheader(
+                f"Retained words: {len(retained_words)} words ({len(retained_words) / len(self.words) * 100:.2f}%)"
+            )
+            st.markdown(
+                "Click on a column to sort by it, place the cursor on the text to display it."
+            )
+            st.dataframe(retained_words)
+
+    def plot_distributions_filtering_parameters(self):
+        st.header("Distributions of the filtering parameters")
+
+        display_distributions = st.checkbox("Display distributions")
+
+        if display_distributions:
+
+            def plot_hist(dataframe, key, num_bins=50):
+                st.subheader(" ".join(key.split("_")))
+                hist_values = dataframe[key].values
+                max_range = np.max(hist_values)
+                hist_values = np.histogram(
+                    hist_values, bins=num_bins, range=(0, max_range)
+                )[0]
+                st.bar_chart(hist_values)
+                st.markdown(f"Each bin is of size: {max_range/num_bins}.")
+
+            for key in list({el[0]: None for el in self.keys}):
+                plot_hist(self.docs, key)
+
+            if not (self.words is None):
+                plot_hist(self.words, "len_word")
+
+    def plot_zipf_law(self):
+        if not (self.words is None):
+            st.header("Zipf's Law")
+
+            display_zipf_law = st.checkbox("Display Zipf's Law")
+
+            if display_zipf_law:
+
+                freq_words = {}
+                for _, row in self.words.iterrows():
+                    freq_words[row["word"]] = freq_words.get(row["word"], 0) + 1
+                freq_words = np.array(list(freq_words.values()))
+                freq_words = -np.sort(-freq_words)
+
+                fig, ax = plt.subplots()
+                ax.loglog(freq_words)
+                ax.set_title("Zipf's Law")
+                ax.set_xlabel("$i$-th most frequent word")
+                ax.set_ylabel("frequency in the documents")
+                st.pyplot(fig)
+
+    def download_data(self):
+        st.header("Download data")
+
+        with open(self.path_data) as json_file:
+            btn = st.download_button(
+                label="Download data as json",
+                data=json_file,
+                file_name="data.json",
+            )
+
+    def visualization(self):
+        self.preamble()
+        self.open_data()
+        self.set_title()
+        self.filtering_of_docs()
+        self.filtering_of_words()
+        self.plot_distributions_filtering_parameters()
+        #self.plot_zipf_law()
+        self.download_data()
+
+
+path_instructions = "./filtering_pipeline_oscar.pdf"
+path_data = "./zh_examples_with_stats.json"
+lang = "Chinese"
+num_docs = 5000
+num_docs_for_words = 500
+max_len_text_display = 10000
+
+visualization = Visualization(
+    path_instructions,
+    path_data,
+    lang,
+    num_docs,
+    num_docs_for_words,
+    max_len_text_display,
+)
+visualization.visualization()