|
|
from pathlib import Path |
|
|
|
|
|
NORM_BASE_SUBMISSION = "114be1f0-5a41-43a5-b4e6-7fb683bc01ec" |
|
|
DIGITS_FOR_VALUES = 3 |
|
|
DIGITS_FOR_ERRORS = 6 |
|
|
|
|
|
DIMENSIONS = { |
|
|
"Segmentation": ['chesapeake', 'sa_crop_type', 'pv4ger_seg', 'cashew', 'neontree', 'nz_cattle'], |
|
|
"Classification": ['pv4ger', 'so2sat', 'brick_kiln', 'big_earth_net', 'eurosat', 'forestnet'], |
|
|
"High Resolution": ['chesapeake', 'pv4ger_seg', 'pv4ger', 'nz_cattle', 'neontree' ], |
|
|
"Medium Resolution": ['brick_kiln', 'big_earth_net', 'eurosat', 'so2sat','sa_crop_type', 'cashew', 'forestnet'], |
|
|
"RGB only": ['pv4ger_seg','chesapeake', 'nz_cattle', 'neontree', 'pv4ger'], |
|
|
"Multi-Spectral": [ 'sa_crop_type', 'cashew','so2sat', 'brick_kiln', 'big_earth_net', 'eurosat', 'forestnet'] |
|
|
} |
|
|
|
|
|
DIMENSION_INFO = { |
|
|
"Segmentation": "datasets for pixel-wise semantic segmentation", |
|
|
"Classification": "single-label and multi-label classification datasets", |
|
|
"High Resolution": "datasets with resolution <= 1 metre", |
|
|
"Medium Resolution": "datasets with 10 metres <= resolution <= 15 metres", |
|
|
"RGB only": "datasets using only Red, Green, and Blue bands", |
|
|
"Multi-Spectral": " datasets with wavelengths beyond the visible spectrum" |
|
|
} |
|
|
|
|
|
|
|
|
DATASETS = [ |
|
|
'big_earth_net', 'so2sat', 'brick_kiln', 'forestnet', 'eurosat', 'pv4ger', |
|
|
'pv4ger_seg', 'chesapeake', 'cashew', 'sa_crop_type', 'nz_cattle', 'neontree' |
|
|
] |
|
|
|
|
|
|
|
|
DATASET_INFO = { |
|
|
"Dataset": [item.replace("_", " ").title() for item in DATASETS], |
|
|
"Description": [ |
|
|
"land cover dataset with multi-label classification in 10 European countries", |
|
|
"local climate zone classification dataset with global coverage", |
|
|
"dataset of brick kilns detected by satellite in Bangladesh", |
|
|
"classification dataset of known forest loss events paired with driver annotations from expert interpreters", |
|
|
"land use/land cover classification dataset", |
|
|
"classification dataset for photo-voltaic systems in Germany", |
|
|
"segmentation dataset for photo-voltaic systems in Germany", |
|
|
"high resolution land cover (segmentation) dataset of Chesapeake Bay, US", |
|
|
"segmentation dataset for cashew plantations in Benin", |
|
|
"segmentation dataset with ground reference crop type labels and multispectral imagery from Germany and South Africa", |
|
|
"segmentation dataset for detecting cows from high resolution aerial images in New Zealand", |
|
|
"segmentation dataset for canopy crown detection and delineation in the US" |
|
|
], |
|
|
"Image Size": ["120 x 120", "32 x 32", "64 x 64", "332 x 332", "64 x 64", "320 x 320", |
|
|
"320 x 320", "256 x 256", "256 x 256","256 x 256", "500 x 500", "400 x 400"], |
|
|
"# Classes": [43, 17, 2, 12, 10, 2, 2, 7, 7, 10, 2, 2], |
|
|
"Train": [20000, 19992, 15063, 6464, 2000, 11814, 3000, 3000, 1350, 3000, 524, 270], |
|
|
"Val": [1000, 986, 999, 989, 1000, 999, 403, 1000, 400, 1000, 66, 94], |
|
|
"Test": [1000, 986, 999, 993, 1000, 999, 403, 1000, 50, 1000, 65, 93], |
|
|
"# Bands": [12, 18, 13, 6, 13, 3, 3, 4, 13, 13, 3, 5], |
|
|
"RGB Res": [10, 10, 10, 15, 10, 0.1, 0.1, 1.0, 10, 10, 0.1, 0.1], |
|
|
"Sensors": ["Sentinel-2","Sentinel-2", "Sentinel-2", "Landsat-8", "Sentinel-2", "RGB", |
|
|
"RGB", "RGBN", "Sentinel-2", "Sentinel-2", "RGB", "RGB"], |
|
|
"Metric": ["Multilabel_F1_Score", "Overall_Accuracy", "Overall_Accuracy", "Overall_Accuracy", "Overall_Accuracy", "Overall_Accuracy", |
|
|
"Multiclass_Jaccard_Index", "Multiclass_Jaccard_Index", "Multiclass_Jaccard_Index", "Multiclass_Jaccard_Index", "Multiclass_Jaccard_Index", "Multiclass_Jaccard_Index"], |
|
|
"Citation": ["Sumbul et al. [2021]", "Zhu et al [2019]", "Lee et al [2021]", "Irvin et al [2020]", "Helber et al [2019]", "Mayer et al [2022]", |
|
|
"Mayer et al [2022]", "Robinson et al [2019]", "Z. et al [2021]", "link", "Abuaiadah and Switzer [2022]", "Weinstein et al. [2021]" |
|
|
], |
|
|
"Hyperlinks": [ |
|
|
"https://arxiv.org/abs/2105.07921", |
|
|
"https://arxiv.org/abs/1912.12171", |
|
|
"https://www.pnas.org/content/118/17/e2018863118", |
|
|
"https://arxiv.org/abs/2011.05479", |
|
|
"https://arxiv.org/abs/1709.00029", |
|
|
"https://www.sciencedirect.com/science/article/pii/S0306261921016937", |
|
|
"https://www.sciencedirect.com/science/article/pii/S0306261921016937", |
|
|
"https://ieeexplore.ieee.org/document/8953207", |
|
|
"https://arxiv.org/abs/2301.00363", |
|
|
"https://source.coop/esa/fusion-competition", |
|
|
"https://doi.org/10.5281/zenodo.5908869", |
|
|
"https://doi.org/10.1371/journal.pcbi.1009180" |
|
|
], |
|
|
"License": ["CDLA-P-1.0", "CC-BY-4.0", "CC-BY-SA 4.0", "CC-BY-4.0", "MIT", "MIT", "MIT", "CDLA-P-1.0", "CC-BY-4.0", "CC-BY-4.0", "CC-BY-4.0", "CC0 1.0"], |
|
|
"Dimensions": [", ".join([dim for dim, data_list in DIMENSIONS.items() if dataset in data_list]) for dataset in DATASETS] |
|
|
} |
|
|
|
|
|
|
|
|
COLUMN_ORDER = { |
|
|
"raw": { |
|
|
"overall_table": ['Overall Mean'] + list(DIMENSIONS.keys()), |
|
|
"dataset_tables": ["Mean"] , |
|
|
"dimension_tables": [] |
|
|
}, |
|
|
"normalized": { |
|
|
"overall_table": ['Overall IQM'] + list(DIMENSIONS.keys()), |
|
|
"dataset_tables": ["IQM"] , |
|
|
"dimension_tables": [] |
|
|
}, |
|
|
"all_tables": ['Model', '# params', 'submission', 'Config Settings'], |
|
|
} |
|
|
|
|
|
root = Path(__file__).parent.resolve() |
|
|
root = "/".join(str(root).split("/")[:-1]) |
|
|
RESULTS_DIR = f"{root}/results" |
|
|
MODEL_INFO_FILE = f"{root}/utils/model_info.json" |
|
|
NORMALIZER_DIR = f"{root}/utils/normalizer" |
|
|
|
|
|
|
|
|
|
|
|
NEW_SUBMISSION_FOLDER = f"{root}/new_submission" |
|
|
CSV_FILE = "results_and_parameters.csv" |
|
|
JSON_FILE = "additional_info.json" |
|
|
NEW_SUBMISSION_COLUMN_INFO = { |
|
|
"string_cols": ['dataset', 'Metric', 'experiment_name', 'partition name', 'backbone', 'decoder','batch_size_selection'], |
|
|
"integer_cols": ['early_stop_patience', 'n_trials', 'Seed', 'data_percentages', 'batch_size'], |
|
|
"float_cols": ['weight_decay', 'lr', 'test metric', ] |
|
|
} |
|
|
NEW_SUBMISSION_COLUMN_NAMES = [] |
|
|
for key, value in NEW_SUBMISSION_COLUMN_INFO.items(): |
|
|
NEW_SUBMISSION_COLUMN_NAMES.extend(value) |
|
|
|
|
|
|
|
|
JSON_FORMAT = { |
|
|
"Paper Link": "N/A", |
|
|
"Code Repository Link ": "N/A", |
|
|
"License": "N/A", |
|
|
"Number of HPO trials": "16", |
|
|
"Additional information about submission": "N/A", |
|
|
"Comments on new models in submission": "N/A", |
|
|
"New model info": |
|
|
[ |
|
|
{ |
|
|
"model_display_name": "TBD", |
|
|
"model_size": "TBD", |
|
|
"unique_backbone_key": "TBD" |
|
|
} |
|
|
] |
|
|
} |
