Upload 2 files

app.py

@@ -21,8 +21,8 @@ with gr.Blocks(theme="NoCrypt/miku@>=1.2.2", fill_width=True, css=css, delete_ca
             with gr.Row():
                 search_civitai_sort = gr.Radio(label="Sort", choices=CIVITAI_SORT, value=CIVITAI_SORT[0])
                 search_civitai_period = gr.Radio(label="Period", choices=CIVITAI_PERIOD, value="Month")
-                search_civitai_limit = gr.Slider(label="Limit", minimum=1, maximum=100, step=1, value=100)
-                search_civitai_page = gr.Slider(label="Page", minimum=1, maximum=100, step=1, value=1)
+                search_civitai_limit = gr.Number(label="Limit", minimum=1, maximum=100, step=1, value=100)
+                search_civitai_page = gr.Number(label="Page", info="If 0, retrieve all pages", minimum=0, maximum=10, step=1, value=1)
             with gr.Row(equal_height=True):
                 search_civitai_query = gr.Textbox(label="Query", placeholder="flux", lines=1)
                 search_civitai_tag = gr.Dropdown(label="Tag", choices=get_civitai_tag(), value=get_civitai_tag()[0], allow_custom_value=True)
@@ -70,6 +70,7 @@ with gr.Blocks(theme="NoCrypt/miku@>=1.2.2", fill_width=True, css=css, delete_ca
         fn=download_civitai,
         inputs=[dl_url, civitai_key, hf_token, uploaded_urls, newrepo_id, newrepo_type, is_private, is_info],
         outputs=[uploaded_urls, urls_md, urls_remain],
+        queue=True,
     )
     gr.on(
         triggers=[search_civitai_submit.click, search_civitai_query.submit, search_civitai_user.submit],
@@ -78,7 +79,7 @@ with gr.Blocks(theme="NoCrypt/miku@>=1.2.2", fill_width=True, css=css, delete_ca
                 search_civitai_period, search_civitai_tag, search_civitai_user, search_civitai_limit, search_civitai_page, search_civitai_gallery, state],
         outputs=[search_civitai_result, search_civitai_desc, search_civitai_submit, search_civitai_query, search_civitai_gallery,
                  search_civitai_result_checkbox, search_civitai_result_info, state],
-        queue=True,
+        queue=False,
         show_api=False,
     )
     search_civitai_result.change(select_civitai_item, [search_civitai_result, state], [search_civitai_desc, search_civitai_json], queue=False, show_api=False)\

civitai_to_hf.py

@@ -153,28 +153,54 @@ CIVITAI_PERIOD = ["AllTime", "Year", "Month", "Week", "Day"]
 
 
 def search_on_civitai(query: str, types: list[str], allow_model: list[str] = [], limit: int = 100,
-                      sort: str = "Highest Rated", period: str = "AllTime", tag: str = "", user: str = "", page: int = 1):
+                      sort: str = "Highest Rated", period: str = "AllTime", tag: str = "", user: str = "", page: int = 1, progress=gr.Progress(track_tqdm=True)):
     user_agent = get_user_agent()
     headers = {'User-Agent': user_agent, 'content-type': 'application/json'}
     base_url = 'https://civitai.com/api/v1/models'
-    params = {'sort': sort, 'period': period, 'limit': int(limit), 'page': int(page),'nsfw': 'true'}
+    params = {'sort': sort, 'period': period, 'limit': int(limit), 'nsfw': 'true'}
     if len(types) != 0: params["types"] = types
     if query: params["query"] = query
     if tag: params["tag"] = tag
     if user: params["username"] = user
+    if page != 0: params["page"] = int(page)
     session = requests.Session()
     retries = Retry(total=5, backoff_factor=1, status_forcelist=[500, 502, 503, 504])
     session.mount("https://", HTTPAdapter(max_retries=retries))
+    rs = []
     try:
-        r = session.get(base_url, params=params, headers=headers, stream=True, timeout=(7.0, 30))
+        if page == 0:
+            progress(0, desc="Searching page 1...")
+            print("Searching page 1...")
+            r = session.get(base_url, params=params | {'page': 1}, headers=headers, stream=True, timeout=(7.0, 30))
+            rs.append(r)
+            if r.ok:
+                json = r.json()
+                next_url = json['metadata']['nextPage'] if 'metadata' in json and 'nextPage' in json['metadata'] else None
+                i = 2
+                while(next_url is not None):
+                    progress(0, desc=f"Searching page {i}...")
+                    print(f"Searching page {i}...")
+                    r = session.get(next_url, headers=headers, stream=True, timeout=(7.0, 30))
+                    rs.append(r)
+                    if r.ok:
+                        json = r.json()
+                        next_url = json['metadata']['nextPage'] if 'metadata' in json and 'nextPage' in json['metadata'] else None
+                    else: next_url = None
+                    i += 1
+        else:
+            progress(0, desc="Searching page 1...")
+            print("Searching page 1...")
+            r = session.get(base_url, params=params, headers=headers, stream=True, timeout=(7.0, 30))
+            rs.append(r)
+    except requests.exceptions.ConnectTimeout:
+        print("Request timed out.")
     except Exception as e:
         print(e)
-        return None
-    else:
-        if not r.ok: return None
+    items = []
+    for r in rs:
+        if not r.ok: continue
         json = r.json()
-        if 'items' not in json: return None
-        items = []
+        if 'items' not in json: continue
         for j in json['items']:
             for model in j['modelVersions']:
                 item = {}
@@ -194,10 +220,10 @@ def search_on_civitai(query: str, types: list[str], allow_model: list[str] = [],
                 item['md'] += f'''Model URL: [https://civitai.com/models/{j["id"]}](https://civitai.com/models/{j["id"]})<br>Model Name: {item["name"]}<br>
                     Creator: {item["creator"]}<br>Tags: {", ".join(item["tags"])}<br>Base Model: {item["base_model"]}<br>Description: {item["description"]}'''
                 items.append(item)
-        return items
+    return items if len(items) > 0 else None
 
 
-def search_civitai(query, types, base_model=[], sort=CIVITAI_SORT[0], period=CIVITAI_PERIOD[0], tag="", user="", limit=100, page=1, gallery=[], state={}):
+def search_civitai(query, types, base_model=[], sort=CIVITAI_SORT[0], period=CIVITAI_PERIOD[0], tag="", user="", limit=100, page=1, gallery=[], state={}, progress=gr.Progress(track_tqdm=True)):
     civitai_last_results = {}
     set_state(state, "civitai_last_choices", [("", "")])
     set_state(state, "civitai_last_gallery", [])