t

app.py

@@ -32,20 +32,6 @@ top_p = gr.inputs.Slider(minimum=0, maximum=1.0,
 # gradio checkbutton
 
 generator = pipeline('text-generation', model='gpt2')
-from transformers import AutoModelForCausalLM, AutoTokenizer
-model = AutoModelForCausalLM.from_pretrained("EleutherAI/gpt-j-6B")
-tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-j-6B")
-
-#  prompt = "In a shocking finding, scientists discovered a herd of unicorns living in a remote, " \
-        #   "previously unexplored valley, in the Andes Mountains. Even more surprising to the " \
-        #   "researchers was the fact that the unicorns spoke perfect English."
-
-# input_ids = tokenizer(prompt, return_tensors="pt").input_ids
-
-# gen_tokens = model.generate(input_ids, do_sample=True, temperature=0.9, max_length=100,)
-# gen_text = tokenizer.batch_decode(gen_tokens)[0]
-
-# gpt_j_generator = pipeline(model='EleutherAI/gpt-j-6B')
 
 
 title = "GPT-J-6B"
@@ -98,10 +84,12 @@ def get_generated_text(generated_text):
     except:
         # recursively loop through generated_text till we get the text
         # don't know if this will work 
-
-        # for i in 
-
-        return generated_text 
+        for gt in generated_text:
+            if 'generated_text' in gt:
+                return gt['generated_text']
+            else:
+                return get_generated_text(gt)
+        # return generated_text 
 
 
 def f(context, temperature, top_p, max_length, model_idx, SPACE_VERIFICATION_KEY):
@@ -113,62 +101,52 @@ def f(context, temperature, top_p, max_length, model_idx, SPACE_VERIFICATION_KEY
         # maybe try "0" instead or 1, or "1"
         # use GPT-J-6B
         if model_idx == 0:
-            # just use regular pipeline models man leave APIs
-            input_ids = tokenizer(context, return_tensors="pt").input_ids
-
-            gen_tokens = model.generate(input_ids, temperature=temperature, max_length=2**11, top_p=top_p, max_new_tokens=max_length, num_return_sequences=1)
-            gen_text = tokenizer.batch_decode(gen_tokens)[0]
-            return gen_text
-            # set_seed(2**31)
-            # generated_text = gpt_j_generator(context, max_length=896, max_new_tokens=max_length, top_p=top_p, temperature=temperature, num_return_sequences=1)
-
-            # return get_generated_text(generated_text)
-            # if main_gpt_j_api_up:
-            #     # for this api, a length of > 250 instantly errors, so use a while loop or something
-            #     # that would fetch results in chunks of 250
-            #     # NOTE change so it uses previous generated input every time
-            #     generated_text = context #""
-            #     while (max_length > 0):
-            #         payload = {"inputs": generated_text, "parameters": {"max_new_tokens": max_length>250 and 250 or max_length, "temperature": temperature, "top_p": top_p}}
-            #         response = requests.request("POST", API_URL, data=json.dumps(payload), headers=headers)
-            #         context = json.loads(response.content.decode("utf-8"))#[0]['generated_text']
-            #         # context = get_generated_text(generated_context)
-            #         # handle inconsistent inference API
-            #         if 'generated_text' in context[0]:
-            #             context = context[0]['generated_text']
-            #         else:
-            #             context = context[0][0]['generated_text']
+            if main_gpt_j_api_up:
+                # for this api, a length of > 250 instantly errors, so use a while loop or something
+                # that would fetch results in chunks of 250
+                # NOTE change so it uses previous generated input every time
+                generated_text = context #""
+                while (max_length > 0):
+                    payload = {"inputs": generated_text, "parameters": {"max_new_tokens": max_length>250 and 250 or max_length, "temperature": temperature, "top_p": top_p}}
+                    response = requests.request("POST", API_URL, data=json.dumps(payload), headers=headers)
+                    context = json.loads(response.content.decode("utf-8"))#[0]['generated_text']
+                    # context = get_generated_text(generated_context)
+                    # handle inconsistent inference API
+                    if 'generated_text' in context[0]:
+                        context = context[0]['generated_text']
+                    else:
+                        context = context[0][0]['generated_text']
                         
-            #         generated_text += context
-            #         max_length -= 250
+                    generated_text += context
+                    max_length -= 250
 
                 # payload = {"inputs": context, "parameters":{
                 #     "max_new_tokens":max_length, "temperature":temperature, "top_p":top_p}}
                 # data = json.dumps(payload)
                 # response = requests.request("POST", API_URL, data=data, headers=headers)
                 # generated_text = json.loads(response.content.decode("utf-8"))[0]['generated_text']
-                # return generated_text
+                return generated_text
 
             # use secondary gpt-j-6B api, as the main one is down
-            # if not secondary_gpt_j_api_up:
-            #     return "ERR: both GPT-J-6B APIs are down, please try again later (will use a third fallback in the future)"
+            if not secondary_gpt_j_api_up:
+                return "ERR: both GPT-J-6B APIs are down, please try again later (will use a third fallback in the future)"
 
             # use fallback API
             # 
             # http://api.vicgalle.net:5000/docs#/default/generate_generate_post
             # https://pythonrepo.com/repo/vicgalle-gpt-j-api-python-natural-language-processing
 
-            # payload = {
-            #     "context": context,
-            #     "token_max_length": max_length,  # 512,
-            #     "temperature": temperature,
-            #     "top_p": top_p,
-            #     "max_time": 120.0
-            # }
+            payload = {
+                "context": context,
+                "token_max_length": max_length,  # 512,
+                "temperature": temperature,
+                "top_p": top_p,
+                "max_time": 120.0
+            }
             
-            # response = requests.post(
-            #     "http://api.vicgalle.net:5000/generate", params=payload).json()
-            # return response['text']
+            response = requests.post(
+                "http://api.vicgalle.net:5000/generate", params=payload).json()
+            return response['text']
         elif model_idx == 1:
             # use GPT-2
             #