Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
File size: 6,668 Bytes
bbb5184 c3c8276 bbb5184 ae0db51 bbb5184 ae0db51 c3c8276 baa2172 7b420fa baa2172 c3c8276 baa2172 7b420fa baa2172 bbb5184 c3c8276 a1180f7 bbb5184 baa2172 bbb5184 c3c8276 bbb5184 a1180f7 bbb5184 ae0db51 a1180f7 ae0db51 c3c8276 a1180f7 ae0db51 fac55e5 c3c8276 fac55e5 c3c8276 fac55e5 c3c8276 ae0db51 fac55e5 bbb5184 c3c8276 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 |
import json
from dataclasses import dataclass
from collections import defaultdict
import logging
logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
@dataclass
class Pasuram:
prabandham_code: str
azhwar_name: str
prabandham_name: str
def get_standardized_prabandham_names() -> list[Pasuram]:
"""
Get a list of prabandham names along with the azhwars who authored them in divya_prabandham,
sorted by the prabandham name (3rd field, index 2).
"""
with open("./data/azhwars.json", "r", encoding="utf-8") as f:
azhwars = json.load(f)
header = azhwars[0]
rows = azhwars[1:]
# Sort by 3rd field (index 2)
rows.sort(key=lambda row: row[2])
final_azhwars = [Pasuram(**dict(zip(header, row))) for row in rows]
return final_azhwars
def get_standardized_azhwar_names() -> list[str]:
"""
Get a list of azhwar names along with the pasurams they have authored in divya_prabandham
"""
with open("./data/azhwars.json", "r", encoding="utf-8") as f:
azhwars = json.load(f)
header = azhwars[0]
rows = azhwars[1:]
final_azhwars = [row[1] for row in rows] ## 2nd field is the azhwar name
return sorted(set(final_azhwars))
def get_standardized_divya_desam_names() -> list[str]:
"""
Get a list of divya desam names in divya_prabandham
"""
with open("./data/divya_desams.json", "r", encoding="utf-8") as f:
divya_desams = json.load(f) # FIXED
selected_fields = [
"title",
"other_names",
"name_ta",
"alwars",
"area",
"state",
"thirukolam",
"direction",
"sampradayam",
"divya_desam",
]
data = [
{key: row[key] for key in selected_fields if key in row}
for row in divya_desams["pageProps"]["hits"]
]
return sorted(set([row["title"] for row in data]))
def reorder_taniyan(collection):
logger.info("reorder_taniyan: started")
# Fetch all docs with ids + metadatas
data = collection.get(include=["metadatas"])
ids = data.get("ids", [])
metas = data.get("metadatas", [])
if not ids or not metas:
logger.warning("reorder_taniyan: no data found in collection")
return
# sort globally by current _global_index
records = sorted(
[(i, m) for i, m in enumerate(metas)],
key=lambda x: x[1].get("_global_index", float("inf")),
)
# group by prabandham_code
grouped = defaultdict(list)
for i, meta in records:
prabandham = meta.get("prabandham_code")
if prabandham:
grouped[prabandham].append((i, meta))
updates = []
global_counter = 1 # running _global_index across the collection
for prabandham, items in grouped.items():
taniyan_items = [
(i, m) for i, m in items if m.get("section_type", "").startswith("taniyan")
]
non_taniyan_items = [
(i, m)
for i, m in items
if not m.get("section_type", "").startswith("taniyan")
]
if not taniyan_items and not non_taniyan_items:
continue
# sort both groups by original _global_index
taniyan_items.sort(key=lambda x: x[1]["_global_index"])
non_taniyan_items.sort(key=lambda x: x[1]["_global_index"])
# --- taniyans first (verse starts from 1) ---
for verse_no, (i, meta) in enumerate(taniyan_items, start=1):
updates.append(
{
"id": ids[i],
"metadata": {
**meta,
"_global_index": global_counter,
"verse": verse_no,
},
}
)
global_counter += 1
# --- non-taniyans continue from their base verse ---
if non_taniyan_items:
base_verse = min(m["verse"] for _, m in non_taniyan_items)
for offset, (i, meta) in enumerate(non_taniyan_items):
updates.append(
{
"id": ids[i],
"metadata": {
**meta,
"_global_index": global_counter,
"verse": base_verse + offset,
},
}
)
global_counter += 1
if updates:
logger.info("reorder_taniyan: updating %d records...", len(updates))
collection.update(
ids=[u["id"] for u in updates],
metadatas=[u["metadata"] for u in updates],
)
logger.info("reorder_taniyan: update complete.")
else:
logger.info("reorder_taniyan: nothing to update")
logger.info("reorder_taniyan: finished")
def delete_taniyan(collection):
logger.info("delete_taniyan: started")
# Fetch all docs (only ids + metadata needed)
data = collection.get(include=["metadatas"])
ids = data["ids"]
metas = data["metadatas"]
# Collect ids where section_type starts with "taniyan"
taniyan_ids = [
ids[i]
for i, meta in enumerate(metas)
if meta.get("section_type", "").startswith("taniyan")
]
if taniyan_ids:
logger.info("delete_taniyan: Deleting %d taniyan records...", len(taniyan_ids))
collection.delete(ids=taniyan_ids)
logger.info("delete_taniyan: Deleted %d taniyan records", len(taniyan_ids))
else:
logger.info("delete_taniyan: No taniyan records found")
logger.info("delete_taniyan: finished")
def get_prabandham_chapter_order_mapping():
chapter_names = [
"Common",
"Thiruppallāṇḍu",
"Periyazvar Thirumozhi",
"Thiruppavai",
"Nachiyar Thirumozhi",
"Perumal Thirumozhi",
"Thiruchandavirutham",
"Thirumalai",
"Thirupalliezhuchi",
"Amalanadipiran",
"Kanninunchiruthambu",
"Periya Thirumozhi",
"Thirukurunthandakam",
"Thirunedumthandakam",
"Muthal Thiruvanthathi",
"Irandam Thiruvanthathi",
"Moonram Thiruvanthathi",
"Nanmukan Thiruvanthathi",
"Thiruvirutham",
"Thiruvasiriyam",
"Periya Thiruvanthathi",
"Thiruvezhukootrarikkai",
"Siriya Thirumadal",
"Periya Thirumadal",
"Thiruvaimozhi",
"Iramanusa Nootranthathi",
]
section_dict = {name: i + 1 for i, name in enumerate(chapter_names)}
return section_dict
if __name__ == "__main__":
logger.info(get_standardized_azhwar_names())
|