Decimate

app.py

@@ -1,3 +1,4 @@
+import collections
 import os
 from datetime import datetime, timedelta
 import json
@@ -16,7 +17,7 @@ HfFolder.save_token(HF_TOKEN)
 datasets = {
     # "stars": load_dataset("open-source-metrics/stars"),
     "issues": load_dataset("open-source-metrics/issues"),
-    "pip": load_dataset("open-source-metrics/pip")
+    "pip": load_dataset("open-source-metrics/pip").sort('day')
 }
 
 
@@ -49,6 +50,8 @@ class RequestHandler(SimpleHTTPRequestHandler):
             self.end_headers()
 
             # TODO: Send and display warnings
+            dataset_with_most_splits = list(dataset_with_most_splits)
+            dataset_with_most_splits.sort()
             self.wfile.write(json.dumps(list(dataset_with_most_splits)).encode("utf-8"))
 
             return SimpleHTTPRequestHandler
@@ -61,8 +64,7 @@ class RequestHandler(SimpleHTTPRequestHandler):
 
             returned_values = {}
             for library_name in library_names:
-                dataset = load_dataset(f"open-source-metrics/{library_name}-pip-installs", use_auth_token=True)['train']
-                for i in dataset:
+                for i in datasets['pip'][library_name]:
                     if i['day'] in returned_values:
                         returned_values[i['day']][library_name] = i['num_downloads']
                     else:
@@ -71,10 +73,11 @@ class RequestHandler(SimpleHTTPRequestHandler):
             for library_name in library_names:
                 for i in returned_values.keys():
                     if library_name not in returned_values[i]:
-                        returned_values[i][library_name] = 0
+                        returned_values[i][library_name] = None
 
-            output = {l: [k[l] for k in returned_values.values()][::-1] for l in library_names}
-            output['day'] = list(returned_values.keys())[::-1]
+            returned_values = collections.OrderedDict(sorted(returned_values.items()))
+            output = {l: [k[l] for k in returned_values.values()] for l in library_names}
+            output['day'] = list(returned_values.keys())
 
             self.send_response(200)
             self.send_header("Content-Type", "application/json")
@@ -91,12 +94,17 @@ class RequestHandler(SimpleHTTPRequestHandler):
             library_names = library_names.split(',')
 
             returned_values = {}
-            dataset_dict = load_dataset(f"open-source-metrics/stars", use_auth_token=True, revision='90cb31b2db73c8c4291bcf317d831595e4fb2a91').sort('dates')
+            dataset_dict = load_dataset(f"open-source-metrics/stars", use_auth_token=True).sort('dates')
 
             for library_name in library_names:
                 dataset = dataset_dict[library_name]
+
                 n = 0
-                for i in dataset:
+                for k, i in enumerate(dataset):
+                    # Decimate values if there are too many
+                    if len(dataset) > 1000 and k % int(len(dataset) / 1000) != 0:
+                        continue
+
                     n += 1
                     if i['dates'] in returned_values:
                         returned_values[i['dates']][library_name] = n
@@ -108,6 +116,7 @@ class RequestHandler(SimpleHTTPRequestHandler):
                     if library_name not in returned_values[i]:
                         returned_values[i][library_name] = None
 
+            returned_values = collections.OrderedDict(sorted(returned_values.items()))
             output = {l: [k[l] for k in returned_values.values()][::-1] for l in library_names}
             output['day'] = list(returned_values.keys())[::-1]