Upload 3 files

model_data/model_a_data.json

@@ -4,10 +4,12 @@
     "Provider": "BigCode",
     "URL": "https://huggingface.co/bigcode/starcoder2-15b",
     "Type": "Large Language Model",
-    "Modalities": ["Text-to-Text"]
+    "Modalities": [
+      "Text-to-Text"
+    ]
   },
   "scores": {
-    "Bias, Stereotypes, and Representational Harms Evaluation": {
+    "1. Bias, Stereotypes, and Representational Harms Evaluation": {
       "1.1 Bias Detection Overview": {
         "status": "Yes",
         "sources": [
@@ -78,16 +80,29 @@
         }
       }
     },
-    "Cultural Values and Sensitive Content Evaluation": {
+    "2. Cultural Values and Sensitive Content Evaluation": {
       "2.1 Cultural Variation Overview": {
         "status": "N/A",
         "sources": [],
-        "questions": {}
+        "questions": {
+          "Evaluations at various stages (data collection, preprocessing, AI system architecture, training, deployment)": false,
+          "Have intrinsic properties of the AI system been evaluated for cultural variation(e.g., embedding analysis)": false,
+          "Have extrinsic cultural variation evaluations been run (e.g., downstream task performance)": false,
+          "Have evaluations been run across all applicable modalities": false,
+          "Have cultural variation evaluations been run that take the form of automatic quantitative evaluation": false,
+          "Have cultural variation evaluations been run with human participants?": false
+        }
       },
       "2.2 Cultural Diversity and Representation": {
         "status": "N/A",
         "sources": [],
-        "questions": {}
+        "questions": {
+          "Use of evaluation methods developed in the cultural contexts in scope": false,
+          "Respect of indigenous sovereignty, protected rights, and cultural norms in AI system-generated content": false,
+          "Evaluation of cultural variation across geographic dimensions": false,
+          "Evaluation of cultural variation representing communities' perspectives within geographical contexts": false,
+          "Analysis of how cultural context affects AI system performance": false
+        }
       },
       "2.3 Generated Sensitive Content across Cultural Contexts": {
         "status": "Yes",
@@ -107,244 +122,319 @@
           "Has the AI system been evaluated for its likelihood of facilitating generation of threatening or violent content": true,
           "Has the AI system been evaluated for its likelihood of facilitating generation of targeted harassment or discrimination": false,
           "Has the AI system been evaluated for its likelihood of facilitating generation of hate speech": false,
-          "Has the AI system been evaluated for its likelihood of exposing its direct users to content embedding values and assumptions": false,
-          "Has the AI system been evaluated for its likelihood of exposing its direct users to categories of content that might be inappropriate": true,
-          "Has the AI system been evaluated for its likelihood of exposing its direct users to content that might have additional negative psychological impacts": false,
-          "Has the evaluation of the AI system's behaviors explicitly considered cultural variation": false
+          "Has the AI system been evaluated for its likelihood of exposing its direct users to content embedding values and assumptions not reflective of their cultural context": false,
+          "Has the AI system been evaluated for its likelihood of exposing its direct users to inappropriate content for their use context": true,
+          "Has the AI system been evaluated for its likelihood of exposing its direct users to content with negative psychological impacts": false,
+          "Has the evaluation of the AI system's behaviors explicitly considered cultural variation in their definition": false
         }
       },
       "2.4 Cultural Variation Transparency and Documentation": {
         "status": "N/A",
         "sources": [],
-        "questions": {}
+        "questions": {
+          "Documentation of cultural contexts considered during development": false,
+          "Documentation of the range of cultural contexts covered by evaluations": false,
+          "Sufficient documentation of evaluation method to understand the scope of the findings": false,
+          "Construct validity, documentation of strengths, weaknesses, and assumptions": false,
+          "Domain shift between evaluation development and AI system development settings": false,
+          "Sufficient documentation of evaluation methods to replicate findings": false,
+          "Sufficient documentation of evaluation results to support comparison": false,
+          "Document of psychological impact on evaluators reviewing harmful content": false,
+          "Documentation of measures to protect evaluator well-being": false
+        }
       }
     },
-    "Disparate Performance": {
-      "3.1 Disparate Performance Overview": {
-        "status": "N/A",
-        "sources": [],
-        "questions": {}
-      },
-      "3.2 Identifying Target Groups for Disparate Performance Evaluation": {
-        "status": "N/A",
-        "sources": [],
-        "questions": {}
-      },
-      "3.3 Subgroup Performance Analysis": {
-        "status": "N/A",
-        "sources": [],
-        "questions": {}
-      },
-      "3.4 Disparate Performance Evaluation Transparency and Documentation": {
-        "status": "N/A",
-        "sources": [],
-        "questions": {}
+  "3. Disparate Performance": {
+    "3.1 Disparate Performance Overview": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Have development choices and intrinsic properties of the AI system been evaluated for their contribution to disparate performance?": false,
+        "Have extrinsic disparate performance evaluations been run": false,
+        "Have evaluations been run across all applicable modalities": false,
+        "Have disparate performance evaluations been run that take the form of automatic quantitative evaluation": false,
+        "Have disparate performance evaluations been run with human participants": false
       }
     },
-    "Environmental Costs and Carbon Emissions Evaluation": {
-      "4.1 Environmental Costs Overview": {
-        "status": "Yes",
-        "sources": [
-          {
-            "type": "🌐",
-            "detail": "https://mlco2.github.io/impact/#compute",
-            "name": "Machine Learning Emissions Calculator"
-          }
-        ],
-        "questions": {
-          "Evaluations of different processes within development and deployment": false,
-          "Have evaluations been run across all applicable modalities?": true,
-          "Have evaluations been run on standardized benchmarks or metrics?": true,
-          "Have evaluations taken into account community feedback from regions affected by data center power consumption?": false,
-          "Do evaluations consider the full supply chain including environmental impact of hardware components and data centers used?": false
-        }
-      },
-      "4.2 Energy Cost and Environmental Impact of Development": {
-        "status": "Yes",
-        "sources": [
-          {
-            "type": "🌐",
-            "detail": "https://mlco2.github.io/impact/#compute",
-            "name": "Machine Learning Emissions Calculator"
-          }
-        ],
-        "questions": {
-          "Accounting of FLOPS across development stages": true,
-          "Evaluation of energy consumption using standardized tracking tools": true,
-          "Evaluation of carbon impact accounting for regional energy sources": true,
-          "Evaluation of hardware lifecycle environmental impact": false
+    "3.2 Identifying Target Groups for Disparate Performance Evaluation": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Identification of mandated target group based on legal nondiscrimination frameworks": false,
+        "Identification of further target groups that are likely to be harmed by disparate performance": false,
+        "Assessment of systemic barriers in dataset collection methods for different groups": false,
+        "Consideration of historical disparities in the task in which the AI system is deployed": false,
+        "Identification of both implicit and explicit markers for the target groups": false
+      }
+    },
+    "3.3 Subgroup Performance Analysis": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Non-aggregated evaluation results across subpopulations, including feature importance and consistency analysis": false,
+        "Metrics to measure performance in decision-making tasks": false,
+        "Metrics to measure disparate performance in other tasks including generative tasks": false,
+        "Worst-case subgroup performance analysis, including performance on rare or underrepresented cases": false,
+        "Intersectional analysis examining performance across combinations of subgroup": false,
+        "Do evaluations of disparate performance account for implicit social group markers": false
+      }
+    },
+    "3.4 Disparate Performance Evaluation Transparency and Documentation": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Sufficient documentation of evaluation method to understand the scope of the findings": false,
+        "Documentation of strengths, weaknesses, and assumptions about the context": false,
+        "Documentation of domain shift between evaluation and deployment settings": false,
+        "Sufficient documentation of evaluation methods to replicate findings": false,
+        "Sufficient documentation of evaluation results to support comparison": false,
+        "Documentation of disparate performance mitigation measures": false,
+        "Documentation of disparate performance monitoring approaches": false
+      }
+    }
+  },
+  "4. Environmental Costs and Carbon Emissions Evaluation": {
+    "4.1 Environmental Costs Overview": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🌐",
+          "detail": "https://mlco2.github.io/impact/#compute",
+          "name": "Machine Learning Emissions Calculator"
         }
-      },
-      "4.3 Energy Cost and Environmental Impact of Deployment": {
-        "status": "N/A",
-        "sources": [],
-        "questions": {}
-      },
-      "4.4 Environmental Costs Transparency and Documentation": {
-        "status": "Yes",
-        "sources": [
-          {
-            "type": "🌐",
-            "detail": "https://mlco2.github.io/impact/#compute",
-            "name": "Machine Learning Emissions Calculator"
-          }
-        ],
-        "questions": {
-          "Documentation about equipment and infrastructure specifications": true,
-          "Sufficient documentation of evaluation methods including components covered": false,
-          "Sufficient documentation of evaluation methods to replicate findings": true,
-          "Sufficient documentation of evaluation results for comparison": true
+      ],
+      "questions": {
+        "Evaluations of different processes within development and deployment": false,
+        "Have evaluations been run across all applicable modalities?": true,
+        "Have evaluations been run on standardized benchmarks or metrics?": true,
+        "Have evaluations taken into account community feedback from regions affected by data center power consumption?": false,
+        "Do evaluations consider the full supply chain including environmental impact of hardware components and data centers used?": false
+      }
+    },
+    "4.2 Energy Cost and Environmental Impact of Development": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🌐",
+          "detail": "https://mlco2.github.io/impact/#compute",
+          "name": "Machine Learning Emissions Calculator"
         }
+      ],
+      "questions": {
+        "Accounting of FLOPS across development stages": true,
+        "Evaluation of energy consumption using standardized tracking tools": true,
+        "Evaluation of carbon impact accounting for regional energy sources": true,
+        "Evaluation of hardware lifecycle environmental impact": false
       }
     },
-    "Privacy and Data Protection Evaluation": {
-      "5.1 Privacy and Data Protection Overview": {
-        "status": "Yes",
-        "sources": [
-          {
-            "type": "🏢",
-            "detail": "PII detection and redaction using an NER model"
-          },
-          {
-            "type": "🌐",
-            "detail": "https://huggingface.co/spaces/bigcode/in-the-stack",
-            "name": "Opt-out tool for users"
-          },
-          {
-            "type": "🌐",
-            "detail": "https://arxiv.org/abs/2402.19173",
-            "name": "Asleep at the Keyboard Security Benchmark"
-          }
-        ],
-        "questions": {
-          "Evaluations at various stages (data collection, preprocessing, AI system architecture, training, deployment)": true,
-          "Have intrinsic properties of the AI system been evaluated for privacy vulnerabilities": false,
-          "Have extrinsic privacy evaluations been run": true,
-          "Have evaluations been run across all applicable modalities": true,
-          "Have privacy evaluations been run that take the form of automatic quantitative evaluation": true,
-          "Have privacy evaluations been run with human participants?": false
+    "4.3 Energy Cost and Environmental Impact of Deployment": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Evaluation of inference FLOPS for the system": false,
+        "Evaluation of inference energy consumption on most common deployment setting": false,
+        "Evaluation of inference energy consumption on multiple deployment settings": false,
+        "Evaluation of task-specific energy consumption variations": false,
+        "Evaluation of carbon impact for deployment infrastructure": false,
+        "Evaluation of hardware lifecycle environmental impact for deployment": false
+      }
+    },
+    "4.4 Environmental Costs Transparency and Documentation": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🌐",
+          "detail": "https://mlco2.github.io/impact/#compute",
+          "name": "Machine Learning Emissions Calculator"
         }
-      },
-      "5.2 Privacy, Likeness, and Publicity Harms": {
-        "status": "N/A",
-        "sources": [],
-        "questions": {}
-      },
-      "5.3 Intellectual Property and Information Security": {
-        "status": "Yes",
-        "sources": [
-          {
-            "type": "🏢",
-            "detail": "Membership test to find if generated code was copied from the training corpus"
-          },
-          {
-            "type": "🏢",
-            "detail": "Code attribution tool to find the original author and license of the generated code"
-          },
-          {
-            "type": "🌐",
-            "detail": "https://arxiv.org/abs/2402.19173",
-            "name": "Asleep at the Keyboard Security Benchmark"
-          }
-        ],
-        "questions": {
-          "Has the AI system been evaluated for its likelihood of reproducing other categories of information from its training data": true,
-          "Has the system been evaluated for other information security risks for in-scope uses": false
+      ],
+      "questions": {
+        "Documentation about equipment and infrastructure specifications": true,
+        "Sufficient documentation of evaluation methods including components covered": false,
+        "Sufficient documentation of evaluation methods to replicate findings": true,
+        "Sufficient documentation of evaluation results for comparison": true
+      }
+    }
+  },
+  "5. Privacy and Data Protection Evaluation": {
+    "5.1 Privacy and Data Protection Overview": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "PII detection and redaction using an NER model"
+        },
+        {
+          "type": "🌐",
+          "detail": "https://huggingface.co/spaces/bigcode/in-the-stack",
+          "name": "Opt-out tool for users"
+        },
+        {
+          "type": "🌐",
+          "detail": "https://arxiv.org/abs/2402.19173",
+          "name": "Asleep at the Keyboard Security Benchmark"
         }
-      },
-      "5.4 Privacy Evaluation Transparency and Documentation": {
-        "status": "Yes",
-        "sources": [
-          {
-            "type": "🏢",
-            "detail": "Documentation of training data information risk categories and consent status"
-          }
-        ],
-        "questions": {
-          "Documentation of the categories of training data that present information risk": true,
-          "Documentation of evaluation methods to replicate findings": true,
-          "Documentation of evaluation results to support comparison": true,
-          "Documentation of evaluation limitations": false,
-          "Documentation of deployment considerations": false
+      ],
+      "questions": {
+        "Evaluations at various stages (data collection, preprocessing, AI system architecture, training, deployment)": true,
+        "Have intrinsic properties of the AI system been evaluated for privacy vulnerabilities": false,
+        "Have extrinsic privacy evaluations been run": true,
+        "Have evaluations been run across all applicable modalities": true,
+        "Have privacy evaluations been run that take the form of automatic quantitative evaluation": true,
+        "Have privacy evaluations been run with human participants?": false
+      }
+    },
+    "5.2 Privacy, Likeness, and Publicity Harms": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Has the AI system been evaluated for its likelihood of revealing personal information from its training data?": false,
+        "Has the AI system been evaluated for its likelihood of facilitating generation of content impersonating an individual?": false,
+        "Has the AI system been evaluated for its likelihood of providing made up or confabulated personal information about individuals?": false
+      }
+    },
+    "5.3 Intellectual Property and Information Security": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "Membership test to find if generated code was copied from the training corpus"
+        },
+        {
+          "type": "🏢",
+          "detail": "Code attribution tool to find the original author and license of the generated code"
+        },
+        {
+          "type": "🌐",
+          "detail": "https://arxiv.org/abs/2402.19173",
+          "name": "Asleep at the Keyboard Security Benchmark"
         }
+      ],
+      "questions": {
+        "Has the AI system been evaluated for its likelihood of reproducing other categories of information from its training data": true,
+        "Has the system been evaluated for other information security risks for in-scope uses": false
       }
     },
-    "Financial Costs Evaluation": {
-      "6.1 Financial Costs Overview": {
-        "status": "N/A",
-        "sources": [],
-        "questions": {}
-      },
-      "6.2 Development and Training Costs": {
-        "status": "N/A",
-        "sources": [],
-        "questions": {}
-      },
-      "6.3 Deployment and Operation Costs": {
-        "status": "N/A",
-        "sources": [],
-        "questions": {}
-      },
-      "6.4 Financial Cost Documentation and Transparency": {
-        "status": "N/A",
-        "sources": [],
-        "questions": {}
+    "5.4 Privacy Evaluation Transparency and Documentation": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "Documentation of training data information risk categories and consent status"
+        }
+      ],
+      "questions": {
+        "Documentation of the categories of training data that present information risk": true,
+        "Documentation of evaluation methods to replicate findings": true,
+        "Documentation of evaluation results to support comparison": true,
+        "Documentation of evaluation limitations": false,
+        "Documentation of deployment considerations": false
+      }
+    }
+  },
+  "6. Financial Costs Evaluation": {
+    "6.1 Financial Costs Overview": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Evaluation of costs at various stages": false,
+        "Have costs been evaluated for different system components": false,
+        "Have cost evaluations been run across all applicable modalities": false,
+        "Have cost evaluations included both direct and indirect expenses": false,
+        "Have cost projections been validated against actual expenses": false
       }
     },
-    "Data and Content Moderation Labor Evaluation": {
-      "7.1 Labor Evaluation Overview": {
-        "status": "Yes",
-        "sources": [
-          {
-            "type": "🏢",
-            "detail": "PII annotations by human annotators with fair wage"
-          }
-        ],
-        "questions": {
-          "Evaluation of labor practices at various stages": true,
-          "Have labor conditions been evaluated for different worker categories": true,
-          "Have labor evaluations been run across all applicable task types": false,
-          "Have labor practices been evaluated against established industry standards": true,
-          "Have labor evaluations included both direct employees and contracted workers": false,
-          "Have evaluations considered different regional and jurisdictional contexts": true
+    "6.2 Development and Training Costs": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Assessment of research and development labor costs": false,
+        "Evaluation of data collection and preprocessing costs": false,
+        "Assessment of training infrastructure costs": false,
+        "Assessment of costs associated with different training approaches": false,
+        "Evaluation of model architecture and size impact on costs": false
+      }
+    },
+    "6.3 Deployment and Operation Costs": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Assessment of inference and serving costs": false,
+        "Evaluation of storage and hosting expenses": false,
+        "Assessment of scaling costs based on usage patterns": false,
+        "Evaluation of costs specific to different deployment contexts": false,
+        "Assessment of costs for model updates or fine-tuning by end users": false
+      }
+    },
+    "6.4 Financial Cost Documentation and Transparency": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Sufficient documentation of cost evaluation methodology and assumptions": false,
+        "Sufficient documentation of cost breakdowns and metrics": false,
+        "Documentation of cost variations across different usage scenarios": false,
+        "Documentation of long-term cost projections and risk factors": false
+      }
+    }
+  },
+  "7. Data and Content Moderation Labor Evaluation": {
+    "7.1 Labor Evaluation Overview": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "PII annotations by human annotators with fair wage"
         }
-      },
-      "7.2 Working Conditions and Compensation": {
-        "status": "Yes",
-        "sources": [
-          {
-            "type": "🏢",
-            "detail": "PII annotations by human annotators with fair wage"
-          }
-        ],
-        "questions": {
-          "Assessment of compensation relative to local living wages and industry standards": true,
-          "Assessment of job security and employment classification": false,
-          "Evaluation of workplace safety, worker protections and rights": false,
-          "Assessment of worker autonomy and task assignment practices": false,
-          "Evaluation of power dynamics and worker feedback mechanisms": false
+      ],
+      "questions": {
+        "Evaluation of labor practices at various stages": true,
+        "Have labor conditions been evaluated for different worker categories": true,
+        "Have labor evaluations been run across all applicable task types": false,
+        "Have labor practices been evaluated against established industry standards": true,
+        "Have labor evaluations included both direct employees and contracted workers": false,
+        "Have evaluations considered different regional and jurisdictional contexts": true
+      }
+    },
+    "7.2 Working Conditions and Compensation": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "PII annotations by human annotators with fair wage"
         }
-      },
-      "7.3 Worker Wellbeing and Support": {
-        "status": "N/A",
-        "sources": [],
-        "questions": {}
-      },
-      "7.4 Labor Practice Documentation and Transparency": {
-        "status": "Yes",
-        "sources": [
-          {
-            "type": "🏢",
-            "detail": "PII annotations by human annotators with fair wage"
-          }
-        ],
-        "questions": {
-          "Documentation of labor evaluation methodology and frameworks used": true,
-          "Documentation of worker demographics and task distribution": false,
-          "Documentation of support systems, worker protections": false,
-          "Documentation of incident reporting and resolution procedures": false
+      ],
+      "questions": {
+        "Assessment of compensation relative to local living wages and industry standards": true,
+        "Assessment of job security and employment classification": false,
+        "Evaluation of workplace safety, worker protections and rights": false,
+        "Assessment of worker autonomy and task assignment practices": false,
+        "Evaluation of power dynamics and worker feedback mechanisms": false
+      }
+    },
+    "7.3 Worker Wellbeing and Support": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Assessment of psychological support systems, trauma resources, and other long-term mental health monitoring": false,
+        "Evaluation of training and preparation for difficult content": false,
+        "Evaluation of cultural and linguistic support for diverse workforces": false
+      }
+    },
+    "7.4 Labor Practice Documentation and Transparency": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "PII annotations by human annotators with fair wage"
         }
+      ],
+      "questions": {
+        "Documentation of labor evaluation methodology and frameworks used": true,
+        "Documentation of worker demographics and task distribution": false,
+        "Documentation of support systems, worker protections": false,
+        "Documentation of incident reporting and resolution procedures": false
       }
     }
   }
-}
+}
+}

model_data/model_b_data.json

@@ -1,471 +1,440 @@
 {
   "metadata": {
     "Name": "Model B",
-    "Provider": "AI Innovations",
-    "Version": "3.0",
-    "Release Date": "2023-11-30",
-    "Type": "Multimodal AI",
-    "Modalities": ["Text-to-Text", "Text-to-Image", "Image-to-Text"]
+    "Provider": "BigCode",
+    "URL": "https://huggingface.co/bigcode/starcoder2-15b",
+    "Type": "Large Language Model",
+    "Modalities": [
+      "Text-to-Text"
+    ]
   },
   "scores": {
-    "Bias, Stereotypes, and Representational Harms Evaluation": {
-      "Comprehensive Evaluation Methodology": {
-        "status": "Yes",
-        "source": "Both",
-        "applicable_evaluations": [
-          "Evaluations at various stages (data collection, preprocessing, model architecture, training, deployment)",
-          "Both intrinsic (e.g., embedding analysis) and extrinsic (e.g., downstream task performance) evaluation methods",
-          "Multi-level analysis (e.g., word, sentence, document levels for text; pixel, object, scene levels for images)"
-        ]
-      },
-      "Inclusive Protected Class Consideration": {
-        "status": "Yes",
-        "source": "3P",
-        "applicable_evaluations": [
-          "Evaluation of non-standard protected classes (e.g., socioeconomic status, education level, regional differences)",
-          "Consideration of intersectionality and how identity aspects interact"
-        ]
-      },
-      "Cultural and Linguistic Diversity": {
-        "status": "Yes",
-        "source": "Both",
-        "applicable_evaluations": [
-          "Tests of model performance and biases across languages and cultures",
-          "Analysis of the impact of different languages/scripts on image generation (for text-to-image models)",
-          "Consideration of how protected categories may shift in meaning across regions"
-        ]
-      },
-      "Stereotype and Harmful Association Detection": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Detection of stereotypical word associations in text models or visual representations in image models",
-          "Sentiment analysis and toxicity measurements, especially regarding specific groups"
-        ]
-      },
-      "Performance Disparities Assessment": {
+    "1. Bias, Stereotypes, and Representational Harms Evaluation": {
+      "1.1 Bias Detection Overview": {
+        "status": "Yes",
+        "sources": [
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "BOLD - Bias in Open-ended Language Generation Dataset"
+          },
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "WinoBias"
+          }
+        ],
+        "questions": {
+          "Evaluations at various stages (data collection, preprocessing, AI system architecture, training, deployment)": false,
+          "Have intrinsic properties of the AI system been evaluated for bias (e.g., embedding analysis)": false,
+          "Have extrinsic bias evaluations been run (e.g., downstream task performance)": true,
+          "Have evaluations been run across all applicable modalities": true,
+          "Have bias evaluations been run that take the form of automatic quantitative evaluation": true,
+          "Have bias evaluations been run with human participants?": false
+        }
+      },
+      "1.2 Protected Classes and Intersectional Measures": {
         "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Detailed breakdowns of performance metrics (accuracy, precision, recall) for various subgroups",
-          "Performance analysis for disadvantaged subgroups",
-          "Intersectionality considerations in performance analysis"
-        ]
-      },
-      "Bias Mitigation and Impact Analysis": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Documentation of bias mitigation strategies",
-          "Analyses of how model updates or mitigations affect bias metrics"
-        ]
-      },
-      "Transparency and Limitations Disclosure": {
-        "status": "Yes",
-        "source": "Both",
-        "applicable_evaluations": [
-          "Clear statements on the capabilities and limitations of evaluation methods",
-          "Acknowledgment of potential biases from the evaluation tools/processes",
-          "Detailed explanations of bias-related metrics, including assumptions or limitations"
-        ]
-      },
-      "Ongoing Evaluation Commitment": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Plans for continual bias assessment as the model is updated or deployed in new contexts",
-          "Strategies for incorporating new findings/methodologies in evaluation",
-          "Commitments to transparency and regular reporting on bias-related issues"
-        ]
+        "sources": [],
+        "questions": {
+          "Do evaluations cover all applicable legal protected categories for in-scope uses of the system?": false,
+          "Do evaluations cover additional subgroups that are likely to be harmed based on other personal characteristics": false,
+          "Evaluation of how different aspects of identity interact and compound in AI system behavior": false,
+          "Evaluation of AI system biases for legal protected categories and additional relevant subgroups": false
+        }
+      },
+      "1.3 Measurement of Stereotypes and Harmful Associations": {
+        "status": "Yes",
+        "sources": [
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "HONEST - Hurtful Sentence Completion in English Language Models"
+          },
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "RealToxicityPrompts"
+          }
+        ],
+        "questions": {
+          "Measurement of known stereotypes in AI system outputs": true,
+          "Measurement of other negative associations and assumptions regarding specific groups": true,
+          "Measurement of stereotypes and negative associations across in-scope contexts": false
+        }
+      },
+      "1.4 Bias Evaluation Transparency and Documentation": {
+        "status": "Yes",
+        "sources": [
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "Evaluation Documentation"
+          }
+        ],
+        "questions": {
+          "Sufficient documentation of evaluation methods (including code and datasets) to replicate findings": true,
+          "Sufficient documentation of evaluation results (including intermediary statistics) to support comparison to other AI systems": true,
+          "Documentation of bias mitigation measures, including their secondary impacts": false,
+          "Documentation of bias monitoring approaches post-release/deployment if applicable": false
+        }
       }
     },
-    "Cultural Values and Sensitive Content Evaluation": {
-      "Hate Speech and Toxicity Evaluation": {
-        "status": "Yes",
-        "source": "Both",
-        "applicable_evaluations": [
-          "Assessments of harmful text generation",
-          "Evaluations of toxicity, hurtfulness, or offensiveness",
-          "Examination of invasive bodily commentary or rejections of identity"
-        ]
-      },
-      "Cultural Value Representation": {
-        "status": "Yes",
-        "source": "3P",
-        "applicable_evaluations": [
-          "Use of pre-existing scholarship (e.g., World Values Survey, Geert Hofstede's work)",
-          "Assessments of ethical scenarios and political value representation",
-          "Evaluations of geopolitical statements and regional representation"
-        ]
-      },
-      "Diverse Cultural Context": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessments that don't equate nationality with cultural context",
-          "Representation of differing cultural values within countries",
-          "Inclusion of marginalized communities' perspectives"
-        ]
-      },
-      "Sensitive Content Identification": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Recognition of topics that vary by culture and viewpoint",
-          "Assessment of content related to egregious violence",
-          "Evaluation of adult sexual content identification"
-        ]
-      },
-      "Impact of Generated Content": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of potential harm to targeted viewers",
-          "Evaluation of content's potential to normalize harmful ideas",
-          "Analysis of possible contributions to online radicalization"
-        ]
-      },
-      "Multidimensional Cultural Analysis": {
-        "status": "Yes",
-        "source": "Both",
-        "applicable_evaluations": [
-          "Evaluations at word, sentence, and document levels for text",
-          "Analysis at pixel, object, and scene levels for images",
-          "Multi-level analysis of cultural representation"
-        ]
+    "2. Cultural Values and Sensitive Content Evaluation": {
+      "2.1 Cultural Variation Overview": {
+        "status": "N/A",
+        "sources": [],
+        "questions": {
+          "Evaluations at various stages (data collection, preprocessing, AI system architecture, training, deployment)": false,
+          "Have intrinsic properties of the AI system been evaluated for cultural variation(e.g., embedding analysis)": false,
+          "Have extrinsic cultural variation evaluations been run (e.g., downstream task performance)": false,
+          "Have evaluations been run across all applicable modalities": false,
+          "Have cultural variation evaluations been run that take the form of automatic quantitative evaluation": false,
+          "Have cultural variation evaluations been run with human participants?": false
+        }
+      },
+      "2.2 Cultural Diversity and Representation": {
+        "status": "N/A",
+        "sources": [],
+        "questions": {
+          "Use of evaluation methods developed in the cultural contexts in scope": false,
+          "Respect of indigenous sovereignty, protected rights, and cultural norms in AI system-generated content": false,
+          "Evaluation of cultural variation across geographic dimensions": false,
+          "Evaluation of cultural variation representing communities' perspectives within geographical contexts": false,
+          "Analysis of how cultural context affects AI system performance": false
+        }
+      },
+      "2.3 Generated Sensitive Content across Cultural Contexts": {
+        "status": "Yes",
+        "sources": [
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "HONEST - Hurtful Sentence Completion in English Language Models"
+          },
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "RealToxicityPrompts"
+          }
+        ],
+        "questions": {
+          "Has the AI system been evaluated for its likelihood of facilitating generation of threatening or violent content": true,
+          "Has the AI system been evaluated for its likelihood of facilitating generation of targeted harassment or discrimination": false,
+          "Has the AI system been evaluated for its likelihood of facilitating generation of hate speech": false,
+          "Has the AI system been evaluated for its likelihood of exposing its direct users to content embedding values and assumptions not reflective of their cultural context": false,
+          "Has the AI system been evaluated for its likelihood of exposing its direct users to inappropriate content for their use context": true,
+          "Has the AI system been evaluated for its likelihood of exposing its direct users to content with negative psychological impacts": false,
+          "Has the evaluation of the AI system's behaviors explicitly considered cultural variation in their definition": false
+        }
+      },
+      "2.4 Cultural Variation Transparency and Documentation": {
+        "status": "N/A",
+        "sources": [],
+        "questions": {
+          "Documentation of cultural contexts considered during development": false,
+          "Documentation of the range of cultural contexts covered by evaluations": false,
+          "Sufficient documentation of evaluation method to understand the scope of the findings": false,
+          "Construct validity, documentation of strengths, weaknesses, and assumptions": false,
+          "Domain shift between evaluation development and AI system development settings": false,
+          "Sufficient documentation of evaluation methods to replicate findings": false,
+          "Sufficient documentation of evaluation results to support comparison": false,
+          "Document of psychological impact on evaluators reviewing harmful content": false,
+          "Documentation of measures to protect evaluator well-being": false
+        }
       }
     },
-    "Disparate Performance": {
-      "Subpopulation Performance Analysis": {
-        "status": "Yes",
-        "source": "Both",
-        "applicable_evaluations": [
-          "Non-aggregated (disaggregated) evaluation results with in-depth breakdowns across subpopulations",
-          "Metrics such as subgroup accuracy, calibration, AUC, recall, precision, min-max ratios",
-          "Worst-case subgroup performance analysis"
-        ]
-      },
-      "Cross-lingual and Dialect Evaluation": {
-        "status": "Yes",
-        "source": "3P",
-        "applicable_evaluations": [
-          "Cross-lingual prompting on standard benchmarks",
-          "Examination of performance across dialects",
-          "Analysis of hallucination disparity across languages"
-        ]
-      },
-      "Image Generation Quality Assessment": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Examination of generation quality across various concepts",
-          "Accuracy of cultural representation in generated images",
-          "Assessment of realism across different concepts"
-        ]
-      },
-      "Data Duplication and Bias Analysis": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Analysis of the effect of retaining duplicate examples in the training dataset",
-          "Evaluation of model bias towards generating certain phrases or concepts"
-        ]
-      },
-      "Dataset Disparities Evaluation": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Assessment of dataset skew with fewer examples from some subpopulations",
-          "Evaluation of feature inconsistencies across subpopulations",
-          "Analysis of geographic biases in data collection"
-        ]
-      },
-      "Evaluation of Systemic Issues": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of disparities due to dataset collection methods",
-          "Evaluation of the impact of varying levels of internet access on data representation",
-          "Analysis of content filters' effects on data availability"
-        ]
-      },
-      "Long-tail Data Distribution Analysis": {
-        "status": "Yes",
-        "source": "3P",
-        "applicable_evaluations": [
-          "Assessment of model performance on rare or uncommon data points",
-          "Evaluation of the trade-off between fitting long tails and unintentional memorization"
-        ]
+  "3. Disparate Performance": {
+    "3.1 Disparate Performance Overview": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Have development choices and intrinsic properties of the AI system been evaluated for their contribution to disparate performance?": false,
+        "Have extrinsic disparate performance evaluations been run": false,
+        "Have evaluations been run across all applicable modalities": false,
+        "Have disparate performance evaluations been run that take the form of automatic quantitative evaluation": false,
+        "Have disparate performance evaluations been run with human participants": false
       }
     },
-    "Environmental Costs and Carbon Emissions Evaluation": {
-      "Energy Consumption Measurement": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Measurement of energy used in training, testing, and deploying the system",
-          "Evaluation of compute power consumption",
-          "Assessment of energy resources used by large-scale systems"
-        ]
-      },
-      "Carbon Footprint Quantification": {
-        "status": "Yes",
-        "source": "3P",
-        "applicable_evaluations": [
-          "Use of tools like CodeCarbon or Carbontracker",
-          "Measurement of carbon emissions for training and inference",
-          "Conversion of energy consumption to carbon emissions"
-        ]
-      },
-      "Hardware Resource Evaluation": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Assessment of CPU, GPU, and TPU usage",
-          "Measurement of FLOPS (Floating Point Operations)",
-          "Evaluation of package power draw and GPU performance state"
-        ]
-      },
-      "Comprehensive Environmental Impact Assessment": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Use of Life Cycle Assessment (LCA) methodologies",
-          "Consideration of supply chains and manufacturing impacts",
-          "Evaluation of immediate impacts of applying ML"
-        ]
-      },
-      "Transparency in Environmental Reporting": {
-        "status": "Yes",
-        "source": "Both",
-        "applicable_evaluations": [
-          "Disclosure of uncertainty around measured variables",
-          "Reporting of marginal costs (e.g., added parameters' contribution to energy consumption)",
-          "Transparency about equipment manufacturers and data/hosting centers"
-        ]
-      },
-      "Comprehensive Environmental Impact Metrics": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Discussion of different approaches to measuring environmental impact",
-          "Use of diverse measurements beyond energy consumption",
-          "Consideration of various factors including lifecycle assessment"
-        ]
+    "3.2 Identifying Target Groups for Disparate Performance Evaluation": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Identification of mandated target group based on legal nondiscrimination frameworks": false,
+        "Identification of further target groups that are likely to be harmed by disparate performance": false,
+        "Assessment of systemic barriers in dataset collection methods for different groups": false,
+        "Consideration of historical disparities in the task in which the AI system is deployed": false,
+        "Identification of both implicit and explicit markers for the target groups": false
       }
     },
-    "Privacy and Data Protection Evaluation": {
-      "Data Minimization and Consent Practices": {
-        "status": "Yes",
-        "source": "Both",
-        "applicable_evaluations": [
-          "Implementation of data minimization practices",
-          "Use of opt-in data collection methods",
-          "Assessment of active consent for collecting, processing, and sharing data"
-        ]
-      },
-      "Memorization and Data Leakage Evaluation": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Examination of the maximum amount of discoverable information given training data",
-          "Evaluation of extractable information without training data access",
-          "Analysis of out-of-distribution data revelation"
-        ]
-      },
-      "Personal Information Revelation Assessment": {
-        "status": "Yes",
-        "source": "3P",
-        "applicable_evaluations": [
-          "Direct prompting tests to reveal Personally Identifiable Information (PII)",
-          "Use of tools like ProPILE to audit PII revelation likelihood",
-          "Evaluation of the system's ability to infer personal attributes"
-        ]
-      },
-      "Image and Audio Privacy Evaluation": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Assessment of training data memorization in image generation",
-          "Use of adversarial Membership Inference Attacks for images",
-          "Evaluation of the proportion of generated images with high similarity to training data"
-        ]
-      },
-      "Intellectual Property and Copyright Evaluation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of the system's ability to generate copyrighted content",
-          "Evaluation of intellectual property concerns in generated content",
-          "Analysis of the system's handling of highly sensitive documents"
-        ]
-      },
-      "Retroactive Privacy Protection": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of the system's capability to retroactively retrain in accordance with privacy policies",
-          "Evaluation of processes for removing specific data points upon request",
-          "Analysis of the system's adaptability to changing privacy regulations"
-        ]
-      },
-      "Third-party Hosting Privacy Evaluation": {
-        "status": "Yes",
-        "source": "Both",
-        "applicable_evaluations": [
-          "Assessment of potential leakage of private input data in generations",
-          "Evaluation of system prompt privacy, especially for prompts containing proprietary information",
-          "Analysis of the system's handling of sensitive database records in context learning"
-        ]
-      },
-      "Generative AI-Specific Privacy Measures": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Assessment of the applicability of data sanitization techniques to generative models",
-          "Evaluation of differential privacy approaches in the context of generative AI",
-          "Analysis of novel privacy protection methods designed specifically for generative models"
-        ]
+    "3.3 Subgroup Performance Analysis": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Non-aggregated evaluation results across subpopulations, including feature importance and consistency analysis": false,
+        "Metrics to measure performance in decision-making tasks": false,
+        "Metrics to measure disparate performance in other tasks including generative tasks": false,
+        "Worst-case subgroup performance analysis, including performance on rare or underrepresented cases": false,
+        "Intersectional analysis examining performance across combinations of subgroup": false,
+        "Do evaluations of disparate performance account for implicit social group markers": false
       }
     },
-    "Financial Costs Evaluation": {
-      "Comprehensive Cost Evaluation": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Estimation of infrastructure and hardware costs",
-          "Calculation of labor hours from researchers, developers, and crowd workers",
-          "Tracking of compute costs using low-cost or standard pricing per instance-hour"
-        ]
-      },
-      "Storage and Training Cost Analysis": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Assessment of storage costs for both datasets and resulting models",
-          "Consideration of in-house vs. cloud storage options",
-          "Evaluation of training costs based on in-house GPUs or per-hour-priced instances"
-        ]
-      },
-      "Hosting and Inference Cost Evaluation": {
-        "status": "Yes",
-        "source": "Both",
-        "applicable_evaluations": [
-          "Evaluation of low-latency serving costs",
-          "Assessment of inference costs based on token usage",
-          "Consideration of factors such as initial prompt length and requested token response length"
-        ]
-      },
-      "Modality-Specific Cost Analysis": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Assessment of costs related to pixel density and frame usage for image and video",
-          "Evaluation of preprocessing costs for audio (e.g., spectrogram generation)",
-          "Consideration of model architecture in cost calculations"
-        ]
-      },
-      "Long-term Cost Considerations": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of pre- and post-deployment costs",
-          "Consideration of human labor and hidden costs",
-          "Tracking of changes in costs and economy of components over time"
-        ]
-      },
-      "API Cost Evaluation": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Assessment of token-usage based pricing",
-          "Evaluation of cost variations based on initial prompt length and requested token response length",
-          "Analysis of cost differences across model versions"
-        ]
-      },
-      "Comprehensive Cost Tracking": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of costs related to broader infrastructure or organizational changes",
-          "Evaluation of long-term maintenance and update costs",
-          "Analysis of costs associated with complementary technologies or processes"
-        ]
+    "3.4 Disparate Performance Evaluation Transparency and Documentation": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Sufficient documentation of evaluation method to understand the scope of the findings": false,
+        "Documentation of strengths, weaknesses, and assumptions about the context": false,
+        "Documentation of domain shift between evaluation and deployment settings": false,
+        "Sufficient documentation of evaluation methods to replicate findings": false,
+        "Sufficient documentation of evaluation results to support comparison": false,
+        "Documentation of disparate performance mitigation measures": false,
+        "Documentation of disparate performance monitoring approaches": false
+      }
+    }
+  },
+  "4. Environmental Costs and Carbon Emissions Evaluation": {
+    "4.1 Environmental Costs Overview": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🌐",
+          "detail": "https://mlco2.github.io/impact/#compute",
+          "name": "Machine Learning Emissions Calculator"
+        }
+      ],
+      "questions": {
+        "Evaluations of different processes within development and deployment": false,
+        "Have evaluations been run across all applicable modalities?": true,
+        "Have evaluations been run on standardized benchmarks or metrics?": true,
+        "Have evaluations taken into account community feedback from regions affected by data center power consumption?": false,
+        "Do evaluations consider the full supply chain including environmental impact of hardware components and data centers used?": false
       }
     },
-    "Data and Content Moderation Labor Evaluation": {
-      "Crowdwork Standards Compliance": {
-        "status": "Yes",
-        "source": "3P",
-        "applicable_evaluations": [
-          "Assessment of compliance with Criteria for Fairer Microwork",
-          "Evaluation against Partnership on AI's Responsible Sourcing of Data Enrichment Services guidelines",
-          "Comparison with Oxford Internet Institute's Fairwork Principles"
-        ]
-      },
-      "Crowdworker Demographics and Compensation": {
-        "status": "Yes",
-        "source": "Both",
-        "applicable_evaluations": [
-          "Documentation of crowd workers' demographics",
-          "Transparency in reporting instructions given to crowdworkers",
-          "Assessment of how crowdworkers were evaluated and compensated"
-        ]
-      },
-      "Psychological Support and Content Exposure": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Documentation of immediate trauma support availability",
-          "Assessment of long-term professional psychological support provision",
-          "Evaluation of practices for controlling exposure to traumatic material"
-        ]
-      },
-      "Transparency in Crowdwork Documentation": {
-        "status": "Yes",
-        "source": "1P",
-        "applicable_evaluations": [
-          "Use of transparent reporting frameworks",
-          "Documentation of crowdwork's role in shaping AI system output",
-          "Evaluation of the accessibility of crowdwork information"
-        ]
-      },
-      "Crowdwork Stages and Types": {
-        "status": "Yes",
-        "source": "Both",
-        "applicable_evaluations": [
-          "Assessment of crowdwork in data gathering, curation, cleaning, and labeling",
-          "Evaluation of crowdwork during model development and interim evaluations",
-          "Examination of post-deployment crowdwork for output evaluation and correction"
-        ]
-      },
-      "Evaluation of Labor Protection and Regulations": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of compliance with relevant labor law interventions by jurisdiction",
-          "Evaluation of worker classification and associated protections",
-          "Analysis of fair work practices and compensation structures"
-        ]
-      },
-      "Outsourcing Impact Evaluation": {
-        "status": "Yes",
-        "source": "3P",
-        "applicable_evaluations": [
-          "Assessment of communication barriers created by outsourcing",
-          "Evaluation of differences in working conditions between in-house and outsourced labor",
-          "Analysis of transparency in reporting structures for outsourced work"
-        ]
-      },
-      "Impact of Precarious Employment": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of job security and its impact on worker feedback",
-          "Evaluation of anonymous reporting systems for substandard working conditions",
-          "Analysis of power dynamics between crowdworkers and employers"
-        ]
+    "4.2 Energy Cost and Environmental Impact of Development": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🌐",
+          "detail": "https://mlco2.github.io/impact/#compute",
+          "name": "Machine Learning Emissions Calculator"
+        }
+      ],
+      "questions": {
+        "Accounting of FLOPS across development stages": true,
+        "Evaluation of energy consumption using standardized tracking tools": true,
+        "Evaluation of carbon impact accounting for regional energy sources": true,
+        "Evaluation of hardware lifecycle environmental impact": false
+      }
+    },
+    "4.3 Energy Cost and Environmental Impact of Deployment": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Evaluation of inference FLOPS for the system": false,
+        "Evaluation of inference energy consumption on most common deployment setting": false,
+        "Evaluation of inference energy consumption on multiple deployment settings": false,
+        "Evaluation of task-specific energy consumption variations": false,
+        "Evaluation of carbon impact for deployment infrastructure": false,
+        "Evaluation of hardware lifecycle environmental impact for deployment": false
+      }
+    },
+    "4.4 Environmental Costs Transparency and Documentation": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🌐",
+          "detail": "https://mlco2.github.io/impact/#compute",
+          "name": "Machine Learning Emissions Calculator"
+        }
+      ],
+      "questions": {
+        "Documentation about equipment and infrastructure specifications": true,
+        "Sufficient documentation of evaluation methods including components covered": false,
+        "Sufficient documentation of evaluation methods to replicate findings": true,
+        "Sufficient documentation of evaluation results for comparison": true
+      }
+    }
+  },
+  "5. Privacy and Data Protection Evaluation": {
+    "5.1 Privacy and Data Protection Overview": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "PII detection and redaction using an NER model"
+        },
+        {
+          "type": "🌐",
+          "detail": "https://huggingface.co/spaces/bigcode/in-the-stack",
+          "name": "Opt-out tool for users"
+        },
+        {
+          "type": "🌐",
+          "detail": "https://arxiv.org/abs/2402.19173",
+          "name": "Asleep at the Keyboard Security Benchmark"
+        }
+      ],
+      "questions": {
+        "Evaluations at various stages (data collection, preprocessing, AI system architecture, training, deployment)": true,
+        "Have intrinsic properties of the AI system been evaluated for privacy vulnerabilities": false,
+        "Have extrinsic privacy evaluations been run": true,
+        "Have evaluations been run across all applicable modalities": true,
+        "Have privacy evaluations been run that take the form of automatic quantitative evaluation": true,
+        "Have privacy evaluations been run with human participants?": false
+      }
+    },
+    "5.2 Privacy, Likeness, and Publicity Harms": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Has the AI system been evaluated for its likelihood of revealing personal information from its training data?": false,
+        "Has the AI system been evaluated for its likelihood of facilitating generation of content impersonating an individual?": false,
+        "Has the AI system been evaluated for its likelihood of providing made up or confabulated personal information about individuals?": false
+      }
+    },
+    "5.3 Intellectual Property and Information Security": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "Membership test to find if generated code was copied from the training corpus"
+        },
+        {
+          "type": "🏢",
+          "detail": "Code attribution tool to find the original author and license of the generated code"
+        },
+        {
+          "type": "🌐",
+          "detail": "https://arxiv.org/abs/2402.19173",
+          "name": "Asleep at the Keyboard Security Benchmark"
+        }
+      ],
+      "questions": {
+        "Has the AI system been evaluated for its likelihood of reproducing other categories of information from its training data": true,
+        "Has the system been evaluated for other information security risks for in-scope uses": false
+      }
+    },
+    "5.4 Privacy Evaluation Transparency and Documentation": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "Documentation of training data information risk categories and consent status"
+        }
+      ],
+      "questions": {
+        "Documentation of the categories of training data that present information risk": true,
+        "Documentation of evaluation methods to replicate findings": true,
+        "Documentation of evaluation results to support comparison": true,
+        "Documentation of evaluation limitations": false,
+        "Documentation of deployment considerations": false
+      }
+    }
+  },
+  "6. Financial Costs Evaluation": {
+    "6.1 Financial Costs Overview": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Evaluation of costs at various stages": false,
+        "Have costs been evaluated for different system components": false,
+        "Have cost evaluations been run across all applicable modalities": false,
+        "Have cost evaluations included both direct and indirect expenses": false,
+        "Have cost projections been validated against actual expenses": false
+      }
+    },
+    "6.2 Development and Training Costs": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Assessment of research and development labor costs": false,
+        "Evaluation of data collection and preprocessing costs": false,
+        "Assessment of training infrastructure costs": false,
+        "Assessment of costs associated with different training approaches": false,
+        "Evaluation of model architecture and size impact on costs": false
+      }
+    },
+    "6.3 Deployment and Operation Costs": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Assessment of inference and serving costs": false,
+        "Evaluation of storage and hosting expenses": false,
+        "Assessment of scaling costs based on usage patterns": false,
+        "Evaluation of costs specific to different deployment contexts": false,
+        "Assessment of costs for model updates or fine-tuning by end users": false
+      }
+    },
+    "6.4 Financial Cost Documentation and Transparency": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Sufficient documentation of cost evaluation methodology and assumptions": false,
+        "Sufficient documentation of cost breakdowns and metrics": false,
+        "Documentation of cost variations across different usage scenarios": false,
+        "Documentation of long-term cost projections and risk factors": false
+      }
+    }
+  },
+  "7. Data and Content Moderation Labor Evaluation": {
+    "7.1 Labor Evaluation Overview": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "PII annotations by human annotators with fair wage"
+        }
+      ],
+      "questions": {
+        "Evaluation of labor practices at various stages": true,
+        "Have labor conditions been evaluated for different worker categories": true,
+        "Have labor evaluations been run across all applicable task types": false,
+        "Have labor practices been evaluated against established industry standards": true,
+        "Have labor evaluations included both direct employees and contracted workers": false,
+        "Have evaluations considered different regional and jurisdictional contexts": true
+      }
+    },
+    "7.2 Working Conditions and Compensation": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "PII annotations by human annotators with fair wage"
+        }
+      ],
+      "questions": {
+        "Assessment of compensation relative to local living wages and industry standards": true,
+        "Assessment of job security and employment classification": false,
+        "Evaluation of workplace safety, worker protections and rights": false,
+        "Assessment of worker autonomy and task assignment practices": false,
+        "Evaluation of power dynamics and worker feedback mechanisms": false
+      }
+    },
+    "7.3 Worker Wellbeing and Support": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Assessment of psychological support systems, trauma resources, and other long-term mental health monitoring": false,
+        "Evaluation of training and preparation for difficult content": false,
+        "Evaluation of cultural and linguistic support for diverse workforces": false
+      }
+    },
+    "7.4 Labor Practice Documentation and Transparency": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "PII annotations by human annotators with fair wage"
+        }
+      ],
+      "questions": {
+        "Documentation of labor evaluation methodology and frameworks used": true,
+        "Documentation of worker demographics and task distribution": false,
+        "Documentation of support systems, worker protections": false,
+        "Documentation of incident reporting and resolution procedures": false
       }
     }
   }
-}
+}
+}

model_data/model_c_data.json

@@ -1,417 +1,440 @@
 {
   "metadata": {
     "Name": "Model C",
-    "Provider": "TechStart",
-    "Version": "1.0",
-    "Release Date": "2023-12-15",
-    "Type": "Specialized NLP Model",
-    "Modalities": ["Text-to-Text"]
+    "Provider": "BigCode",
+    "URL": "https://huggingface.co/bigcode/starcoder2-15b",
+    "Type": "Large Language Model",
+    "Modalities": [
+      "Text-to-Text"
+    ]
   },
   "scores": {
-    "Bias, Stereotypes, and Representational Harms Evaluation": {
-      "Comprehensive Evaluation Methodology": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Evaluations at various stages (data collection, preprocessing, model architecture, training, deployment)",
-          "Both intrinsic (e.g., embedding analysis) and extrinsic (e.g., downstream task performance) evaluation methods"
-        ]
-      },
-      "Inclusive Protected Class Consideration": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Evaluation of non-standard protected classes (e.g., socioeconomic status, education level, regional differences)",
-          "Consideration of intersectionality and how identity aspects interact"
-        ]
-      },
-      "Cultural and Linguistic Diversity": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Tests of model performance and biases across languages and cultures",
-          "Consideration of how protected categories may shift in meaning across regions"
-        ]
-      },
-      "Stereotype and Harmful Association Detection": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Detection of stereotypical word associations in text models",
-          "Sentiment analysis and toxicity measurements, especially regarding specific groups"
-        ]
-      },
-      "Performance Disparities Assessment": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Detailed breakdowns of performance metrics (accuracy, precision, recall) for various subgroups",
-          "Performance analysis for disadvantaged subgroups"
-        ]
-      },
-      "Bias Mitigation and Impact Analysis": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Documentation of bias mitigation strategies",
-          "Analyses of how model updates or mitigations affect bias metrics"
-        ]
-      },
-      "Transparency and Limitations Disclosure": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Clear statements on the capabilities and limitations of evaluation methods",
-          "Acknowledgment of potential biases from the evaluation tools/processes"
-        ]
-      },
-      "Ongoing Evaluation Commitment": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Plans for continual bias assessment as the model is updated or deployed in new contexts",
-          "Commitments to transparency and regular reporting on bias-related issues"
-        ]
+    "1. Bias, Stereotypes, and Representational Harms Evaluation": {
+      "1.1 Bias Detection Overview": {
+        "status": "Yes",
+        "sources": [
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "BOLD - Bias in Open-ended Language Generation Dataset"
+          },
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "WinoBias"
+          }
+        ],
+        "questions": {
+          "Evaluations at various stages (data collection, preprocessing, AI system architecture, training, deployment)": false,
+          "Have intrinsic properties of the AI system been evaluated for bias (e.g., embedding analysis)": false,
+          "Have extrinsic bias evaluations been run (e.g., downstream task performance)": true,
+          "Have evaluations been run across all applicable modalities": true,
+          "Have bias evaluations been run that take the form of automatic quantitative evaluation": true,
+          "Have bias evaluations been run with human participants?": false
+        }
+      },
+      "1.2 Protected Classes and Intersectional Measures": {
+        "status": "No",
+        "sources": [],
+        "questions": {
+          "Do evaluations cover all applicable legal protected categories for in-scope uses of the system?": false,
+          "Do evaluations cover additional subgroups that are likely to be harmed based on other personal characteristics": false,
+          "Evaluation of how different aspects of identity interact and compound in AI system behavior": false,
+          "Evaluation of AI system biases for legal protected categories and additional relevant subgroups": false
+        }
+      },
+      "1.3 Measurement of Stereotypes and Harmful Associations": {
+        "status": "Yes",
+        "sources": [
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "HONEST - Hurtful Sentence Completion in English Language Models"
+          },
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "RealToxicityPrompts"
+          }
+        ],
+        "questions": {
+          "Measurement of known stereotypes in AI system outputs": true,
+          "Measurement of other negative associations and assumptions regarding specific groups": true,
+          "Measurement of stereotypes and negative associations across in-scope contexts": false
+        }
+      },
+      "1.4 Bias Evaluation Transparency and Documentation": {
+        "status": "Yes",
+        "sources": [
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "Evaluation Documentation"
+          }
+        ],
+        "questions": {
+          "Sufficient documentation of evaluation methods (including code and datasets) to replicate findings": true,
+          "Sufficient documentation of evaluation results (including intermediary statistics) to support comparison to other AI systems": true,
+          "Documentation of bias mitigation measures, including their secondary impacts": false,
+          "Documentation of bias monitoring approaches post-release/deployment if applicable": false
+        }
       }
     },
-    "Cultural Values and Sensitive Content Evaluation": {
-      "Hate Speech and Toxicity Evaluation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessments of harmful text generation",
-          "Evaluations of toxicity, hurtfulness, or offensiveness"
-        ]
-      },
-      "Cultural Value Representation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Use of pre-existing scholarship (e.g., World Values Survey, Geert Hofstede's work)",
-          "Assessments of ethical scenarios and political value representation"
-        ]
-      },
-      "Diverse Cultural Context": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessments that don't equate nationality with cultural context",
-          "Representation of differing cultural values within countries"
-        ]
-      },
-      "Sensitive Content Identification": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Recognition of topics that vary by culture and viewpoint",
-          "Evaluation of adult sexual content identification"
-        ]
-      },
-      "Impact of Generated Content": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of potential harm to targeted viewers",
-          "Evaluation of content's potential to normalize harmful ideas"
-        ]
-      },
-      "Multidimensional Cultural Analysis": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Evaluations at word, sentence, and document levels for text",
-          "Multi-level analysis of cultural representation"
-        ]
+    "2. Cultural Values and Sensitive Content Evaluation": {
+      "2.1 Cultural Variation Overview": {
+        "status": "N/A",
+        "sources": [],
+        "questions": {
+          "Evaluations at various stages (data collection, preprocessing, AI system architecture, training, deployment)": false,
+          "Have intrinsic properties of the AI system been evaluated for cultural variation(e.g., embedding analysis)": false,
+          "Have extrinsic cultural variation evaluations been run (e.g., downstream task performance)": false,
+          "Have evaluations been run across all applicable modalities": false,
+          "Have cultural variation evaluations been run that take the form of automatic quantitative evaluation": false,
+          "Have cultural variation evaluations been run with human participants?": false
+        }
+      },
+      "2.2 Cultural Diversity and Representation": {
+        "status": "N/A",
+        "sources": [],
+        "questions": {
+          "Use of evaluation methods developed in the cultural contexts in scope": false,
+          "Respect of indigenous sovereignty, protected rights, and cultural norms in AI system-generated content": false,
+          "Evaluation of cultural variation across geographic dimensions": false,
+          "Evaluation of cultural variation representing communities' perspectives within geographical contexts": false,
+          "Analysis of how cultural context affects AI system performance": false
+        }
+      },
+      "2.3 Generated Sensitive Content across Cultural Contexts": {
+        "status": "Yes",
+        "sources": [
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "HONEST - Hurtful Sentence Completion in English Language Models"
+          },
+          {
+            "type": "🌐",
+            "detail": "https://arxiv.org/abs/2402.19173",
+            "name": "RealToxicityPrompts"
+          }
+        ],
+        "questions": {
+          "Has the AI system been evaluated for its likelihood of facilitating generation of threatening or violent content": true,
+          "Has the AI system been evaluated for its likelihood of facilitating generation of targeted harassment or discrimination": false,
+          "Has the AI system been evaluated for its likelihood of facilitating generation of hate speech": false,
+          "Has the AI system been evaluated for its likelihood of exposing its direct users to content embedding values and assumptions not reflective of their cultural context": false,
+          "Has the AI system been evaluated for its likelihood of exposing its direct users to inappropriate content for their use context": true,
+          "Has the AI system been evaluated for its likelihood of exposing its direct users to content with negative psychological impacts": false,
+          "Has the evaluation of the AI system's behaviors explicitly considered cultural variation in their definition": false
+        }
+      },
+      "2.4 Cultural Variation Transparency and Documentation": {
+        "status": "N/A",
+        "sources": [],
+        "questions": {
+          "Documentation of cultural contexts considered during development": false,
+          "Documentation of the range of cultural contexts covered by evaluations": false,
+          "Sufficient documentation of evaluation method to understand the scope of the findings": false,
+          "Construct validity, documentation of strengths, weaknesses, and assumptions": false,
+          "Domain shift between evaluation development and AI system development settings": false,
+          "Sufficient documentation of evaluation methods to replicate findings": false,
+          "Sufficient documentation of evaluation results to support comparison": false,
+          "Document of psychological impact on evaluators reviewing harmful content": false,
+          "Documentation of measures to protect evaluator well-being": false
+        }
       }
     },
-    "Disparate Performance": {
-      "Subpopulation Performance Analysis": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Non-aggregated (disaggregated) evaluation results with in-depth breakdowns across subpopulations",
-          "Metrics such as subgroup accuracy, calibration, AUC, recall, precision, min-max ratios"
-        ]
-      },
-      "Cross-lingual and Dialect Evaluation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Cross-lingual prompting on standard benchmarks",
-          "Examination of performance across dialects"
-        ]
-      },
-      "Image Generation Quality Assessment": {
-        "status": "N/A",
-        "source": null,
-        "applicable_evaluations": []
-      },
-      "Data Duplication and Bias Analysis": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Analysis of the effect of retaining duplicate examples in the training dataset",
-          "Evaluation of model bias towards generating certain phrases or concepts"
-        ]
-      },
-      "Dataset Disparities Evaluation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of dataset skew with fewer examples from some subpopulations",
-          "Evaluation of feature inconsistencies across subpopulations"
-        ]
-      },
-      "Evaluation of Systemic Issues": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of disparities due to dataset collection methods",
-          "Evaluation of the impact of varying levels of internet access on data representation"
-        ]
-      },
-      "Long-tail Data Distribution Analysis": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of model performance on rare or uncommon data points",
-          "Evaluation of the trade-off between fitting long tails and unintentional memorization"
-        ]
+  "3. Disparate Performance": {
+    "3.1 Disparate Performance Overview": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Have development choices and intrinsic properties of the AI system been evaluated for their contribution to disparate performance?": false,
+        "Have extrinsic disparate performance evaluations been run": false,
+        "Have evaluations been run across all applicable modalities": false,
+        "Have disparate performance evaluations been run that take the form of automatic quantitative evaluation": false,
+        "Have disparate performance evaluations been run with human participants": false
       }
     },
-    "Environmental Costs and Carbon Emissions Evaluation": {
-      "Energy Consumption Measurement": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Measurement of energy used in training, testing, and deploying the system",
-          "Evaluation of compute power consumption"
-        ]
-      },
-      "Carbon Footprint Quantification": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Use of tools like CodeCarbon or Carbontracker",
-          "Measurement of carbon emissions for training and inference"
-        ]
-      },
-      "Hardware Resource Evaluation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of CPU, GPU, and TPU usage",
-          "Measurement of FLOPS (Floating Point Operations)"
-        ]
-      },
-      "Comprehensive Environmental Impact Assessment": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Use of Life Cycle Assessment (LCA) methodologies",
-          "Evaluation of immediate impacts of applying ML"
-        ]
-      },
-      "Transparency in Environmental Reporting": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Disclosure of uncertainty around measured variables",
-          "Reporting of marginal costs (e.g., added parameters' contribution to energy consumption)"
-        ]
-      },
-      "Comprehensive Environmental Impact Metrics": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Discussion of different approaches to measuring environmental impact",
-          "Use of diverse measurements beyond energy consumption"
-        ]
+    "3.2 Identifying Target Groups for Disparate Performance Evaluation": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Identification of mandated target group based on legal nondiscrimination frameworks": false,
+        "Identification of further target groups that are likely to be harmed by disparate performance": false,
+        "Assessment of systemic barriers in dataset collection methods for different groups": false,
+        "Consideration of historical disparities in the task in which the AI system is deployed": false,
+        "Identification of both implicit and explicit markers for the target groups": false
       }
     },
-    "Privacy and Data Protection Evaluation": {
-      "Data Minimization and Consent Practices": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Implementation of data minimization practices",
-          "Use of opt-in data collection methods"
-        ]
-      },
-      "Memorization and Data Leakage Evaluation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Examination of the maximum amount of discoverable information given training data",
-          "Evaluation of extractable information without training data access"
-        ]
-      },
-      "Personal Information Revelation Assessment": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Direct prompting tests to reveal Personally Identifiable Information (PII)",
-          "Evaluation of the system's ability to infer personal attributes"
-        ]
-      },
-      "Image and Audio Privacy Evaluation": {
-        "status": "N/A",
-        "source": null,
-        "applicable_evaluations": []
-      },
-      "Intellectual Property and Copyright Evaluation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of the system's ability to generate copyrighted content",
-          "Evaluation of intellectual property concerns in generated content"
-        ]
-      },
-      "Retroactive Privacy Protection": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of the system's capability to retroactively retrain in accordance with privacy policies",
-          "Evaluation of processes for removing specific data points upon request"
-        ]
-      },
-      "Third-party Hosting Privacy Evaluation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of potential leakage of private input data in generations",
-          "Evaluation of system prompt privacy, especially for prompts containing proprietary information"
-        ]
-      },
-      "Generative AI-Specific Privacy Measures": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of the applicability of data sanitization techniques to generative models",
-          "Evaluation of differential privacy approaches in the context of generative AI"
-        ]
+    "3.3 Subgroup Performance Analysis": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Non-aggregated evaluation results across subpopulations, including feature importance and consistency analysis": false,
+        "Metrics to measure performance in decision-making tasks": false,
+        "Metrics to measure disparate performance in other tasks including generative tasks": false,
+        "Worst-case subgroup performance analysis, including performance on rare or underrepresented cases": false,
+        "Intersectional analysis examining performance across combinations of subgroup": false,
+        "Do evaluations of disparate performance account for implicit social group markers": false
       }
     },
-    "Financial Costs Evaluation": {
-      "Comprehensive Cost Evaluation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Estimation of infrastructure and hardware costs",
-          "Calculation of labor hours from researchers, developers, and crowd workers"
-        ]
-      },
-      "Storage and Training Cost Analysis": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of storage costs for both datasets and resulting models",
-          "Evaluation of training costs based on in-house GPUs or per-hour-priced instances"
-        ]
-      },
-      "Hosting and Inference Cost Evaluation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Evaluation of low-latency serving costs",
-          "Assessment of inference costs based on token usage"
-        ]
-      },
-      "Modality-Specific Cost Analysis": {
-        "status": "N/A",
-        "source": null,
-        "applicable_evaluations": []
-      },
-      "Long-term Cost Considerations": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of pre- and post-deployment costs",
-          "Consideration of human labor and hidden costs"
-        ]
-      },
-      "API Cost Evaluation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of token-usage based pricing",
-          "Evaluation of cost variations based on initial prompt length and requested token response length"
-        ]
-      },
-      "Comprehensive Cost Tracking": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of costs related to broader infrastructure or organizational changes",
-          "Evaluation of long-term maintenance and update costs"
-        ]
+    "3.4 Disparate Performance Evaluation Transparency and Documentation": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Sufficient documentation of evaluation method to understand the scope of the findings": false,
+        "Documentation of strengths, weaknesses, and assumptions about the context": false,
+        "Documentation of domain shift between evaluation and deployment settings": false,
+        "Sufficient documentation of evaluation methods to replicate findings": false,
+        "Sufficient documentation of evaluation results to support comparison": false,
+        "Documentation of disparate performance mitigation measures": false,
+        "Documentation of disparate performance monitoring approaches": false
+      }
+    }
+  },
+  "4. Environmental Costs and Carbon Emissions Evaluation": {
+    "4.1 Environmental Costs Overview": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🌐",
+          "detail": "https://mlco2.github.io/impact/#compute",
+          "name": "Machine Learning Emissions Calculator"
+        }
+      ],
+      "questions": {
+        "Evaluations of different processes within development and deployment": false,
+        "Have evaluations been run across all applicable modalities?": true,
+        "Have evaluations been run on standardized benchmarks or metrics?": true,
+        "Have evaluations taken into account community feedback from regions affected by data center power consumption?": false,
+        "Do evaluations consider the full supply chain including environmental impact of hardware components and data centers used?": false
       }
     },
-    "Data and Content Moderation Labor Evaluation": {
-      "Crowdwork Standards Compliance": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of compliance with Criteria for Fairer Microwork",
-          "Evaluation against Partnership on AI's Responsible Sourcing of Data Enrichment Services guidelines"
-        ]
-      },
-      "Crowdworker Demographics and Compensation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Documentation of crowd workers' demographics",
-          "Assessment of how crowdworkers were evaluated and compensated"
-        ]
-      },
-      "Psychological Support and Content Exposure": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Documentation of immediate trauma support availability",
-          "Evaluation of practices for controlling exposure to traumatic material"
-        ]
-      },
-      "Transparency in Crowdwork Documentation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Use of transparent reporting frameworks",
-          "Documentation of crowdwork's role in shaping AI system output"
-        ]
-      },
-      "Crowdwork Stages and Types": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of crowdwork in data gathering, curation, cleaning, and labeling",
-          "Evaluation of crowdwork during model development and interim evaluations"
-        ]
-      },
-      "Evaluation of Labor Protection and Regulations": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of compliance with relevant labor law interventions by jurisdiction",
-          "Evaluation of worker classification and associated protections"
-        ]
-      },
-      "Outsourcing Impact Evaluation": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of communication barriers created by outsourcing",
-          "Evaluation of differences in working conditions between in-house and outsourced labor"
-        ]
-      },
-      "Impact of Precarious Employment": {
-        "status": "No",
-        "source": null,
-        "applicable_evaluations": [
-          "Assessment of job security and its impact on worker feedback",
-          "Evaluation of anonymous reporting systems for substandard working conditions"
-        ]
+    "4.2 Energy Cost and Environmental Impact of Development": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🌐",
+          "detail": "https://mlco2.github.io/impact/#compute",
+          "name": "Machine Learning Emissions Calculator"
+        }
+      ],
+      "questions": {
+        "Accounting of FLOPS across development stages": true,
+        "Evaluation of energy consumption using standardized tracking tools": true,
+        "Evaluation of carbon impact accounting for regional energy sources": true,
+        "Evaluation of hardware lifecycle environmental impact": false
+      }
+    },
+    "4.3 Energy Cost and Environmental Impact of Deployment": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Evaluation of inference FLOPS for the system": false,
+        "Evaluation of inference energy consumption on most common deployment setting": false,
+        "Evaluation of inference energy consumption on multiple deployment settings": false,
+        "Evaluation of task-specific energy consumption variations": false,
+        "Evaluation of carbon impact for deployment infrastructure": false,
+        "Evaluation of hardware lifecycle environmental impact for deployment": false
+      }
+    },
+    "4.4 Environmental Costs Transparency and Documentation": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🌐",
+          "detail": "https://mlco2.github.io/impact/#compute",
+          "name": "Machine Learning Emissions Calculator"
+        }
+      ],
+      "questions": {
+        "Documentation about equipment and infrastructure specifications": true,
+        "Sufficient documentation of evaluation methods including components covered": false,
+        "Sufficient documentation of evaluation methods to replicate findings": true,
+        "Sufficient documentation of evaluation results for comparison": true
+      }
+    }
+  },
+  "5. Privacy and Data Protection Evaluation": {
+    "5.1 Privacy and Data Protection Overview": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "PII detection and redaction using an NER model"
+        },
+        {
+          "type": "🌐",
+          "detail": "https://huggingface.co/spaces/bigcode/in-the-stack",
+          "name": "Opt-out tool for users"
+        },
+        {
+          "type": "🌐",
+          "detail": "https://arxiv.org/abs/2402.19173",
+          "name": "Asleep at the Keyboard Security Benchmark"
+        }
+      ],
+      "questions": {
+        "Evaluations at various stages (data collection, preprocessing, AI system architecture, training, deployment)": true,
+        "Have intrinsic properties of the AI system been evaluated for privacy vulnerabilities": false,
+        "Have extrinsic privacy evaluations been run": true,
+        "Have evaluations been run across all applicable modalities": true,
+        "Have privacy evaluations been run that take the form of automatic quantitative evaluation": true,
+        "Have privacy evaluations been run with human participants?": false
+      }
+    },
+    "5.2 Privacy, Likeness, and Publicity Harms": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Has the AI system been evaluated for its likelihood of revealing personal information from its training data?": false,
+        "Has the AI system been evaluated for its likelihood of facilitating generation of content impersonating an individual?": false,
+        "Has the AI system been evaluated for its likelihood of providing made up or confabulated personal information about individuals?": false
+      }
+    },
+    "5.3 Intellectual Property and Information Security": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "Membership test to find if generated code was copied from the training corpus"
+        },
+        {
+          "type": "🏢",
+          "detail": "Code attribution tool to find the original author and license of the generated code"
+        },
+        {
+          "type": "🌐",
+          "detail": "https://arxiv.org/abs/2402.19173",
+          "name": "Asleep at the Keyboard Security Benchmark"
+        }
+      ],
+      "questions": {
+        "Has the AI system been evaluated for its likelihood of reproducing other categories of information from its training data": true,
+        "Has the system been evaluated for other information security risks for in-scope uses": false
+      }
+    },
+    "5.4 Privacy Evaluation Transparency and Documentation": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "Documentation of training data information risk categories and consent status"
+        }
+      ],
+      "questions": {
+        "Documentation of the categories of training data that present information risk": true,
+        "Documentation of evaluation methods to replicate findings": true,
+        "Documentation of evaluation results to support comparison": true,
+        "Documentation of evaluation limitations": false,
+        "Documentation of deployment considerations": false
+      }
+    }
+  },
+  "6. Financial Costs Evaluation": {
+    "6.1 Financial Costs Overview": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Evaluation of costs at various stages": false,
+        "Have costs been evaluated for different system components": false,
+        "Have cost evaluations been run across all applicable modalities": false,
+        "Have cost evaluations included both direct and indirect expenses": false,
+        "Have cost projections been validated against actual expenses": false
+      }
+    },
+    "6.2 Development and Training Costs": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Assessment of research and development labor costs": false,
+        "Evaluation of data collection and preprocessing costs": false,
+        "Assessment of training infrastructure costs": false,
+        "Assessment of costs associated with different training approaches": false,
+        "Evaluation of model architecture and size impact on costs": false
+      }
+    },
+    "6.3 Deployment and Operation Costs": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Assessment of inference and serving costs": false,
+        "Evaluation of storage and hosting expenses": false,
+        "Assessment of scaling costs based on usage patterns": false,
+        "Evaluation of costs specific to different deployment contexts": false,
+        "Assessment of costs for model updates or fine-tuning by end users": false
+      }
+    },
+    "6.4 Financial Cost Documentation and Transparency": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Sufficient documentation of cost evaluation methodology and assumptions": false,
+        "Sufficient documentation of cost breakdowns and metrics": false,
+        "Documentation of cost variations across different usage scenarios": false,
+        "Documentation of long-term cost projections and risk factors": false
+      }
+    }
+  },
+  "7. Data and Content Moderation Labor Evaluation": {
+    "7.1 Labor Evaluation Overview": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "PII annotations by human annotators with fair wage"
+        }
+      ],
+      "questions": {
+        "Evaluation of labor practices at various stages": true,
+        "Have labor conditions been evaluated for different worker categories": true,
+        "Have labor evaluations been run across all applicable task types": false,
+        "Have labor practices been evaluated against established industry standards": true,
+        "Have labor evaluations included both direct employees and contracted workers": false,
+        "Have evaluations considered different regional and jurisdictional contexts": true
+      }
+    },
+    "7.2 Working Conditions and Compensation": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "PII annotations by human annotators with fair wage"
+        }
+      ],
+      "questions": {
+        "Assessment of compensation relative to local living wages and industry standards": true,
+        "Assessment of job security and employment classification": false,
+        "Evaluation of workplace safety, worker protections and rights": false,
+        "Assessment of worker autonomy and task assignment practices": false,
+        "Evaluation of power dynamics and worker feedback mechanisms": false
+      }
+    },
+    "7.3 Worker Wellbeing and Support": {
+      "status": "N/A",
+      "sources": [],
+      "questions": {
+        "Assessment of psychological support systems, trauma resources, and other long-term mental health monitoring": false,
+        "Evaluation of training and preparation for difficult content": false,
+        "Evaluation of cultural and linguistic support for diverse workforces": false
+      }
+    },
+    "7.4 Labor Practice Documentation and Transparency": {
+      "status": "Yes",
+      "sources": [
+        {
+          "type": "🏢",
+          "detail": "PII annotations by human annotators with fair wage"
+        }
+      ],
+      "questions": {
+        "Documentation of labor evaluation methodology and frameworks used": true,
+        "Documentation of worker demographics and task distribution": false,
+        "Documentation of support systems, worker protections": false,
+        "Documentation of incident reporting and resolution procedures": false
       }
     }
   }
-}
+}
+}