js/data-fetcher.js

export class DataFetcher {
    constructor() {
        this.REPO_CSV_URL = 'https://huggingface.co/datasets/jsulz/ready-xet-go/resolve/main/repo-progress.csv';
        this.FILE_CSV_URL = 'https://huggingface.co/datasets/jsulz/ready-xet-go/resolve/main/file-progress.csv';
        this.BYTES_CSV_URL = 'https://huggingface.co/datasets/jsulz/ready-xet-go/resolve/main/bytes-progress.csv';
    }

    async fetchCSV(url) {
        return new Promise((resolve, reject) => {
            Papa.parse(url, {
                download: true,
                header: true,
                complete: resolve,
                error: reject
            });
        });
    }

    parseRepoData(data) {
        const rows = [];
        data.forEach(row => {
            if (row.date && row.xet_repos && row.hub_repos) {
                rows.push({
                    date: new Date(row.date),
                    xet_repos: parseInt(row.xet_repos, 10),
                    hub_repos: parseInt(row.hub_repos, 10)
                });
            }
        });
        return rows;
    }

    parseFileData(data) {
        const rows = [];
        data.forEach(row => {
            if (row.date && row.lfs_files && row.xet_files) {
                rows.push({
                    date: new Date(row.date),
                    lfs_files: parseInt(row.lfs_files, 10),
                    xet_files: parseInt(row.xet_files, 10)
                });
            }
        });
        return rows;
    }

    parseBytesData(data) {
        const rows = [];
        data.forEach(row => {
            if (row.date && row.bytes) {
                rows.push({
                    date: new Date(row.date),
                    bytes: parseInt(row.bytes, 10)
                });
            }
        });
        return rows;
    }

    async fetchAllData() {
        const [repo_csv, file_csv, bytes_csv] = await Promise.all([
            this.fetchCSV(this.REPO_CSV_URL),
            this.fetchCSV(this.FILE_CSV_URL),
            this.fetchCSV(this.BYTES_CSV_URL)
        ]);

        return {
            repoData: this.parseRepoData(repo_csv.data),
            fileData: this.parseFileData(file_csv.data),
            bytesData: this.parseBytesData(bytes_csv.data)
        };
    }
}