t

app.py

@@ -195,38 +195,16 @@ def f(context, temperature, top_p, max_length, model_idx, SPACE_VERIFICATION_KEY
             # TODO use fallback gpt-2 inference api for this as well 
             # TODO or just make it an option in the menu "GPT-2 inference"
         elif model_idx == 2: 
-            DISTIL_GPT2_API_URL = "https://api-inference.huggingface.co/models/distilgpt2"
-            # generated_text=""
-            # while (max_length > 0):
-            #     # NOTE see original implementation above for gpt-J-6B
-            #     payload = {"inputs": context, "parameters": {"max_new_tokens": 250, "temperature": temperature, "top_p": top_p}}
-            #     response = requests.request("POST", DISTIL_GPT2_API_URL, data=json.dumps(payload), headers=headers)
-            #     context = json.loads(response.content.decode("utf-8"))
-                    
-            #     context = get_generated_text(context)
-
-            #     generated_text += context
-            #     max_length -= 250
+            url = "https://api-inference.huggingface.co/models/distilgpt2"
             generated_text = ""#context #""
-            while len(generated_text) < max_length:#(max_length > 0): NOTE NOTE commented out this line and added new check
-                # context becomes the previous generated context
-                # NOTE I've set return_full_text to false, see how this plays out
-                # change max_length from max_length>250 and 250 or max_length to 250
+            while len(generated_text) < max_length:
                 payload = {"inputs": context, "parameters": {"return_full_text":False, "max_new_tokens": 250, "temperature": temperature, "top_p": top_p}}
-                response = requests.request("POST", DISTIL_GPT2_API_URL, data=json.dumps(payload), headers=headers)
-                context = json.loads(response.content.decode("utf-8"))#[0]['generated_text']
-                # context = get_generated_text(generated_context)
-                
-                # handle inconsistent inference API
-                # if 'generated_text' in context[0]:
-                #     context = context[0]['generated_text']
-                # else:
-                #     context = context[0][0]['generated_text']
-                    
+                response = requests.request("POST", url, data=json.dumps(payload), headers=headers)
+                context = json.loads(response.content.decode("utf-8"))
                 context = get_generated_text(context).strip()
 
                 generated_text += context
-
+            return generated_text
                 # payload = {"inputs": context, "parameters":{
                 #     "max_new_tokens":max_length, "temperature":temperature, "top_p":top_p}}
                 # data = json.dumps(payload)
@@ -248,6 +226,7 @@ def f(context, temperature, top_p, max_length, model_idx, SPACE_VERIFICATION_KEY
         else:
             url = "https://api-inference.huggingface.co/models/EleutherAI/gpt-neo-2.7B"
             generated_text = ""#context #""
+            # NOTE we're actually using max_new_tokens and min_new_tokens
             while len(generated_text) < max_length:
                 payload = {"inputs": context, "parameters": {"return_full_text":False, "max_new_tokens": 250, "temperature": temperature, "top_p": top_p}}
                 response = requests.request("POST", url, data=json.dumps(payload), headers=headers)