t

app.py

@@ -194,7 +194,25 @@ def f(context, temperature, top_p, max_length, model_idx, SPACE_VERIFICATION_KEY
             # TODO use fallback gpt-2 inference api for this as well 
             # TODO or just make it an option in the menu "GPT-2 inference"
         else: 
-            url = "https://api-inference.huggingface.co/models/distilgpt2"
+            API_URL = "https://api-inference.huggingface.co/models/distilgpt2"
+            generated_text=""
+            while (max_length > 0):
+                    # NOTE see original implementation above for gpt-J-6B
+                    payload = {"inputs": context, "parameters": {"max_new_tokens": max_length>250 and 250 or max_length, "temperature": temperature, "top_p": top_p}}
+                    response = requests.request("POST", API_URL, data=json.dumps(payload), headers=headers)
+                    context = json.loads(response.content.decode("utf-8"))
+                        
+                    context = get_generated_text(context)
+
+                    generated_text += context
+                    max_length -= 250
+
+                # payload = {"inputs": context, "parameters":{
+                #     "max_new_tokens":max_length, "temperature":temperature, "top_p":top_p}}
+                # data = json.dumps(payload)
+                # response = requests.request("POST", API_URL, data=data, headers=headers)
+                # generated_text = json.loads(response.content.decode("utf-8"))[0]['generated_text']
+                return generated_text#context #_context+generated_text
 
     except Exception as e:
         return f"error with idx{model_idx}: "+str(e)