File size: 3,723 Bytes
8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 88b3855 8e64e24 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 |
#!/bin/sh
# Check environment variables
if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
echo "Missing required environment variables HF_TOKEN or DATASET_ID"
exit 1
fi
# Upload backup
cat > /tmp/hf_sync.py << 'EOL'
from huggingface_hub import HfApi
import sys
import os
def manage_backups(api, repo_id, max_files=50):
files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
backup_files = [f for f in files if f.startswith('webui_backup_') and f.endswith('.db')]
backup_files.sort()
if len(backup_files) >= max_files:
files_to_delete = backup_files[:(len(backup_files) - max_files + 1)]
for file_to_delete in files_to_delete:
try:
api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type="dataset")
print(f'Deleted old backup: {file_to_delete}')
except Exception as e:
print(f'Error deleting {file_to_delete}: {str(e)}')
def upload_backup(file_path, file_name, token, repo_id):
api = HfApi(token=token)
try:
api.upload_file(
path_or_fileobj=file_path,
path_in_repo=file_name,
repo_id=repo_id,
repo_type="dataset"
)
print(f"Successfully uploaded {file_name}")
manage_backups(api, repo_id)
except Exception as e:
print(f"File upload error: {str(e)}")
# Download latest backup
def download_latest_backup(token, repo_id):
try:
api = HfApi(token=token)
files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
backup_files = [f for f in files if f.startswith('webui_backup_') and f.endswith('.db')]
if not backup_files:
print("Backup file not found")
return
latest_backup = sorted(backup_files)[-1]
filepath = api.hf_hub_download(
repo_id=repo_id,
filename=latest_backup,
repo_type="dataset"
)
if filepath and os.path.exists(filepath):
os.makedirs('./data', exist_ok=True)
os.system(f'cp "{filepath}" ./data/webui.db')
print(f"Successfully restored backup from {latest_backup}")
except Exception as e:
print(f"Error downloading backup: {str(e)}")
if __name__ == "__main__":
action = sys.argv[1]
token = sys.argv[2]
repo_id = sys.argv[3]
if action == "upload":
file_path = sys.argv[4]
file_name = sys.argv[5]
upload_backup(file_path, file_name, token, repo_id)
elif action == "download":
download_latest_backup(token, repo_id)
EOL
# Download latest backup on first startup
echo "Downloading latest backup from HuggingFace..."
python3 /tmp/hf_sync.py download "${HF_TOKEN}" "${DATASET_ID}"
# Sync function
sync_data() {
while true; do
echo "Starting sync process $(date)"
if [ -f "./data/webui.db" ]; then
timestamp=$(date +%Y%m%d_%H%M%S)
backup_file="webui_backup_${timestamp}.db"
# Copy database file
cp ./data/webui.db "/tmp/${backup_file}"
echo "Uploading backup to HuggingFace..."
python3 /tmp/hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
rm -f "/tmp/${backup_file}"
else
echo "Database file does not exist, waiting for next sync..."
fi
SYNC_INTERVAL=${SYNC_INTERVAL:-7200}
echo "Next sync will be in ${SYNC_INTERVAL} seconds..."
sleep $SYNC_INTERVAL
done
}
# Start sync process in background
sync_data & |