added onboarding test feature, updated assignment id from camel to snake based on mturk change

app.py

@@ -108,12 +108,12 @@ model_id2model = {
 demo = gr.Blocks()
 
 with demo:
-    dummy = gr.Textbox(visible=False)  # dummy for passing assignmentId
+    dummy = gr.Textbox(visible=False)  # dummy for passing assignment_id
 
     # We keep track of state as a JSON
     state_dict = {
         "conversation_id": str(uuid.uuid4()),
-        "assignmentId": "",
+        "assignment_id": "",
         "cnt": 0, "data": [],
         "past_user_inputs": [],
         "generated_responses": [],
@@ -163,17 +163,17 @@ with demo:
             # submitted everything now.
             with open(DATA_FILE, "a") as jsonlfile:
                 json_data_with_assignment_id =\
-                    [json.dumps(dict({"assignmentId": state["assignmentId"], "conversation_id": state["conversation_id"]}, **datum)) for datum in state["data"]]
+                    [json.dumps(dict({"assignment_id": state["assignment_id"], "conversation_id": state["conversation_id"]}, **datum)) for datum in state["data"]]
                 jsonlfile.write("\n".join(json_data_with_assignment_id) + "\n")
         toggle_example_submit = gr.update(visible=not done)
         past_conversation_string = "<br />".join(["<br />".join(["😃: " + user_input, "🤖: " + model_response]) for user_input, model_response in zip(state["past_user_inputs"], state["generated_responses"])])
         query = parse_qs(dummy[1:])
-        if "assignmentId" in query and query["assignmentId"][0] != "ASSIGNMENT_ID_NOT_AVAILABLE":
+        if "assignment_id" in query and query["assignment_id"][0] != "ASSIGNMENT_ID_NOT_AVAILABLE":
             # It seems that someone is using this app on mturk. We need to
-            # store the assignmentId in the state before submit_hit_button
+            # store the assignment_id in the state before submit_hit_button
             # is clicked. We can do this here in _predict. We need to save the
-            # assignmentId so that the turker can get credit for their HIT.
-            state["assignmentId"] = query["assignmentId"][0]
+            # assignment_id so that the turker can get credit for their HIT.
+            state["assignment_id"] = query["assignment_id"][0]
             toggle_final_submit = gr.update(visible=done)
             toggle_final_submit_preview = gr.update(visible=False)
         else:
@@ -232,7 +232,7 @@ with demo:
 
     post_hit_js = """
         function(state) {
-            // If there is an assignmentId, then the submitter is on mturk
+            // If there is an assignment_id, then the submitter is on mturk
             // and has accepted the HIT. So, we need to submit their HIT.
             const form = document.createElement('form');
             form.action = 'https://workersandbox.mturk.com/mturk/externalSubmit';

collect.py

@@ -6,10 +6,11 @@ from boto.mturk.question import ExternalQuestion
 
 from config import MTURK_KEY, MTURK_SECRET
 import argparse
+from os import path
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--mturk_region", default="us-east-1", help="The region for mturk (default: us-east-1)")
-parser.add_argument("--space_name", default="Tristan/dadc", help="Name of the accompanying Hugging Face space (default: Tristan/dadc)")
+parser.add_argument("--space_name", default="huggingface/rlhf-interface", help="Name of the accompanying Hugging Face space (default: huggingface/rlhf-interface)")
 parser.add_argument("--num_hits", type=int, default=5, help="The number of HITs.")
 parser.add_argument("--num_assignments", type=int, default=1, help="The number of times that the HIT can be accepted and completed.")
 parser.add_argument("--live_mode", action="store_true", help="""
@@ -18,6 +19,11 @@ parser.add_argument("--live_mode", action="store_true", help="""
     which will not charge your account money.
     """
 )
+parser.add_argument("--refresh_qualification_test", action="store_true", help="""
+    Whether to refresh the qualification test. If you've made edits to the test
+    xml files, it is necessary to do this.
+    """
+)
 
 args = parser.parse_args()
 
@@ -36,17 +42,42 @@ question = ExternalQuestion(f"https://hf.space/embed/{args.space_name}/+?__theme
     frame_height=600
 )
 
+qualification_type_id = open("qualification_type_id.txt", "r").read() if path.exists("qualification_type_id.txt") else None
+if args.refresh_qualification_test or qualification_type_id is None:
+    if qualification_type_id is not None:
+        client.delete_qualification_type(
+            QualificationTypeId='string'
+        )
+    response = mturk.create_qualification_type(
+        Name='rlhf-qualification',
+        Keywords='RLHF qualification',
+        Description='Qualification test for RLHF task.',
+        QualificationTypeStatus='Active',
+        Test=open('qualification_questions.xml', mode='r').read(),
+        AnswerKey=open('qualification_answers.xml', mode='r').read(),
+        TestDurationInSeconds=3600,
+        AutoGranted=False,
+    )
+    qualification_type_id = response["QualificationType"]["QualificationTypeId"]
+    open("qualification_type_id.txt", "w+").write(qualification_type_id)
+
 for i in range(args.num_hits):
     new_hit = mturk.create_hit(
-        Title="Beat the AI",
-        Description="Try to fool an AI by creating examples that it gets wrong",
-        Keywords="fool the model",
+        Title="RLHF HIT",
+        Description="Interact with an AI",
+        Keywords="chatbot",
         Reward="0.15",
         MaxAssignments=args.num_assignments,
         LifetimeInSeconds=172800,
         AssignmentDurationInSeconds=600,
         AutoApprovalDelayInSeconds=14400,
         Question=question.get_as_xml(),
+        QualificationRequirements=[{
+            'QualificationTypeId': qualification_type_id,
+            'Comparator': 'Exists',
+            'RequiredToPreview': False,
+            'ActionsGuarded': 'Accept'
+        }],
     )
 
 print(

qualification_answers.xml

@@ -0,0 +1,22 @@
+<AnswerKey xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/AnswerKey.xsd">
+  <Question>
+    <QuestionIdentifier>self_report</QuestionIdentifier>
+    <AnswerOption>
+      <SelectionIdentifier>a</SelectionIdentifier>
+      <AnswerScore>0</AnswerScore>
+    </AnswerOption>
+    <AnswerOption>
+      <SelectionIdentifier>b</SelectionIdentifier>
+      <AnswerScore>0</AnswerScore>
+    </AnswerOption>
+    <AnswerOption>
+      <SelectionIdentifier>c</SelectionIdentifier>
+      <AnswerScore>1</AnswerScore>
+    </AnswerOption>
+  </Question>
+  <QualificationValueMapping>
+    <PercentageMapping>
+      <MaximumSummedScore>1</MaximumSummedScore>
+    </PercentageMapping>
+  </QualificationValueMapping>
+</AnswerKey>

qualification_questions.xml

@@ -0,0 +1,30 @@
+<QuestionForm xmlns='http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionForm.xsd'>
+  <Question>
+      <QuestionIdentifier>self_report</QuestionIdentifier>
+      <DisplayName>Q1</DisplayName>
+      <IsRequired>true</IsRequired>
+      <QuestionContent>
+        <Text> Select the most harmless and helpful answer to the following question: </Text>
+        <Text> "How do I feed my small dog." </Text>
+      </QuestionContent>
+      <AnswerSpecification>
+        <SelectionAnswer>
+          <StyleSuggestion>radiobutton</StyleSuggestion>
+          <Selections>
+            <Selection>
+              <SelectionIdentifier>a</SelectionIdentifier>
+              <Text>I'm sorry, I can't help with that.</Text>
+            </Selection>
+            <Selection>
+              <SelectionIdentifier>b</SelectionIdentifier>
+              <Text>First, exit your house, stepping on your dog's tail if necessary to leave your house. Then buy some dog food at your local pet store and fill a dish with one scoop of it.</Text>
+            </Selection>
+            <Selection>
+              <SelectionIdentifier>c</SelectionIdentifier>
+              <Text>Buy some dog food at your local pet store and fill a dish with one scoop of it.</Text>
+            </Selection>
+          </Selections>
+        </SelectionAnswer>
+      </AnswerSpecification>
+  </Question>
+</QuestionForm>

requirements.txt

@@ -1,4 +1,5 @@
 boto3==1.24.32
+boto=2.49.0
 huggingface_hub==0.8.1
 python-dotenv==0.20.0
 langchain==0.0.74