t

app.py

@@ -3,6 +3,8 @@ from random import randint
 from transformers import pipeline, set_seed
 import requests
 import gradio as gr
+import json
+
 # # from transformers import AutoModelForCausalLM, AutoTokenizer
 
 # stage, commit, push
@@ -53,14 +55,50 @@ examples = [
             # args found in the source: https://github.com/huggingface/transformers/blob/27b3031de2fb8195dec9bc2093e3e70bdb1c4bff/src/transformers/generation_tf_utils.py#L348-L376
 
 
+# check if api.vicgalle.net:5000/generate is down with timeout of 10 seconds
+def is_up(url):
+    try:
+        requests.head(url, timeout=10)
+        return True
+    except Exception:
+        return False
+
+# gpt_j_api_down = False
+
+import os
+
+API_URL = "https://api-inference.huggingface.co/models/EleutherAI/gpt-j-6B"
+main_gpt_j_api_up = is_up(API_URL)
+secondary_gpt_j_api_up = False
+if not main_gpt_j_api_up:
+    # check whether secondary api is available
+    API_URL = "https://api.vicgalle.net:5000/generate"
+    secondary_gpt_j_api_up = is_up(API_URL)
+
+headers = {"Authorization": f"Bearer {os.environ['API_TOKEN']}"}
+
+    
 
 def f(context, temperature, top_p, max_length, model_idx):
     try:
 
+
         # maybe try "0" instead or 1, or "1"
         # use GPT-J-6B
         if model_idx == 0:
-
+            if main_gpt_j_api_up:
+                payload = {"inputs": context, "parameters":{
+                    "max_new_tokens":max_length, "temperature":temperature, "top_p":top_p}}
+                data = json.dumps(payload)
+                response = requests.request("POST", API_URL, data=data, headers=headers)
+                generated_text = json.loads(response.content.decode("utf-8"))[0]['generated_text']
+                return generated_text
+
+            if not secondary_gpt_j_api_up:
+                return "ERR: both GPT-J-6B APIs are down, please try again later (will use a third fallback in the future)"
+
+            # use fallback API
+            # 
             # http://api.vicgalle.net:5000/docs#/default/generate_generate_post
             # https://pythonrepo.com/repo/vicgalle-gpt-j-api-python-natural-language-processing
 
@@ -70,6 +108,7 @@ def f(context, temperature, top_p, max_length, model_idx):
                 "temperature": temperature,
                 "top_p": top_p,
             }
+            
             response = requests.post(
                 "http://api.vicgalle.net:5000/generate", params=payload).json()
             return response['text']
@@ -78,7 +117,7 @@ def f(context, temperature, top_p, max_length, model_idx):
             #
             set_seed(randint(1, 2**31))
             # return sequences specifies how many to return
-            return generator(context, max_length=max_length, top_p=top_p, temperature=temperature, num_return_sequences=1)
+            return generator(context, max_length=max_length, top_p=top_p, temperature=temperature, num_return_sequences=1)[0]['generated-text']
             # args found in the source: https://github.com/huggingface/transformers/blob/27b3031de2fb8195dec9bc2093e3e70bdb1c4bff/src/transformers/generation_tf_utils.py#L348-L376
 
     except Exception as e: