WOW

app.py

@@ -17,12 +17,13 @@ HfFolder.save_token(HF_TOKEN)
 datasets = {
     "stars": load_dataset("open-source-metrics/preprocessed_stars"),
     "issues": load_dataset("open-source-metrics/preprocessed_issues"),
-    "pip": load_dataset("open-source-metrics/pip").sort('day'),
+    "pip": load_dataset("open-source-metrics/preprocessed_pip").sort('day'),
 }
 
 external_datasets = {
     "pip": load_dataset("open-source-metrics/pip-external").sort('day'),
-    "stars": load_dataset("open-source-metrics/stars-external")
+    "stars": load_dataset("open-source-metrics/stars-external"),
+    "issues": load_dataset("open-source-metrics/issues-external")
 }
 
 
@@ -53,23 +54,21 @@ class RequestHandler(SimpleHTTPRequestHandler):
             return SimpleHTTPRequestHandler.do_GET(self)
 
         if self.path.startswith("/initialize"):
-            dataset_keys = {k: set(v.keys()) for k, v in datasets.items()}
-            dataset_with_most_splits = max([d for d in dataset_keys.values()], key=len)
+            dataset_with_most_splits = max(datasets['stars'].column_names.values(), key=len)
+
+            if 'day' in dataset_with_most_splits:
+                dataset_with_most_splits.remove('day')
 
             external_dataset_keys = {k: set(v.keys()) for k, v in external_datasets.items()}
             external_dataset_with_most_splits = max([d for d in external_dataset_keys.values()], key=len)
 
+            for external in external_dataset_with_most_splits:
+                dataset_with_most_splits.remove(external)
+
             warnings = []
 
             print("Initializing ...")
 
-            # for k, v in dataset_keys.items():
-            #     if len(v) < len(dataset_with_most_splits):
-            #         warnings.append(
-            #             f"The {k} dataset does not contain all splits. Missing: {dataset_with_most_splits - v}."
-            #             f"\nSelecting that split to show the pip install numbers will not work."
-            #         )
-
             for k, v in external_dataset_keys.items():
                 if len(v) < len(external_dataset_with_most_splits):
                     warnings.append(
@@ -96,54 +95,35 @@ class RequestHandler(SimpleHTTPRequestHandler):
         if self.path.startswith("/retrievePipInstalls"):
             errors = []
             library_names, options = parse_name_and_options(self.path)
+            cumulated = '1' in options
+            week_over_week = '2' in options
 
-            if '1' in options:
-                returned_values = {}
-                for library_name in library_names:
-                    ds = None
-                    if library_name in datasets['pip']:
-                        ds = datasets['pip'][library_name]
-                    elif library_name in external_datasets['pip']:
-                        ds = external_datasets['pip'][library_name]
-                    else:
-                        errors.append(f"No {library_name} found in internal or external datasets.")
-
-                    for i in ds:
-                        if i['day'] in returned_values:
-                            returned_values[i['day']]['Cumulated'] += i['num_downloads']
-                        else:
-                            returned_values[i['day']] = {'Cumulated': i['num_downloads']}
-
-                library_names = ['Cumulated']
+            def sum_of_lists(lists):
+                def _sum(items):
+                    while None in items:
+                        items.remove(None)
+                    return sum(items)
 
+                return [_sum(list(a)) for a in zip(*lists)]
+
+            if week_over_week:
+                if cumulated:
+                    cumulated_dict = {
+                        'Cumulated': sum_of_lists([v for k, v in datasets['pip']['wow'].to_dict().items() if k in library_names]),
+                        'day': datasets['pip']['wow'].to_dict()['day']
+                    }
+                    return self.response(cumulated_dict)
+                else:
+                    return self.response({k: v for k, v in datasets['pip']['wow'].to_dict().items() if k in library_names + ['day']})
             else:
-                returned_values = {}
-                for library_name in library_names:
-
-                    if library_name in datasets['pip']:
-                        ds = datasets['pip'][library_name]
-                    elif library_name in external_datasets['pip']:
-                        ds = external_datasets['pip'][library_name]
-                    else:
-                        errors.append(f"No {library_name} found in internal or external datasets for pip.")
-                        return {'errors': errors}
-
-                    for i in ds:
-                        if i['day'] in returned_values:
-                            returned_values[i['day']][library_name] = i['num_downloads']
-                        else:
-                            returned_values[i['day']] = {library_name: i['num_downloads']}
-
-                for library_name in library_names:
-                    for i in returned_values.keys():
-                        if library_name not in returned_values[i]:
-                            returned_values[i][library_name] = None
-
-            returned_values = collections.OrderedDict(sorted(returned_values.items()))
-            output = {l: [k[l] for k in returned_values.values()] for l in library_names}
-            output['day'] = list(returned_values.keys())
-
-            return self.response(output)
+                if cumulated:
+                    cumulated_dict = {
+                        'Cumulated': sum_of_lists([v for k, v in datasets['pip']['raw'].to_dict().items() if k in library_names]),
+                        'day': datasets['pip']['raw'].to_dict()['day']
+                    }
+                    return self.response(cumulated_dict)
+                else:
+                    return self.response({k: v for k, v in datasets['pip']['raw'].to_dict().items() if k in library_names + ['day']})
 
         if self.path.startswith("/retrieveStars"):
             library_names, options = parse_name_and_options(self.path)

index.js

@@ -192,8 +192,11 @@ const initialize = async () => {
     }
 
     addOption('pip', "Cumulated");
+    addOption('pip', "Week over week");
+
     addOption('issue', "Exclude org members");
     addOption('issue', "Week over week");
+
     addOption('stars', "Week over week");
 
     const fetchButton = createButton('Fetch', inferJson, () => {