Clean freebase_qa, craigslist_bargains, health_fact, hyperpartisan_news_detection (#660)

* Added metrics to all tasks. Reduced original task assignment to minimal closed book QA prompt

* Updated all QA prompts to indicate original task

* craigslist_bargins:
- updated non-original tasks to use answer choices and added accuracy as the metric
- some prompts generate a lot of extra whitespace, which I assume can be dealt with in preprocessing

health_fact:
- added answer choices and metrics to relevant prompts
- cleaned up prompt names

* hyperpartisan_news_detection (byarticle, bypublisher):
- cleaned up prompt names (they are all paraphrases of the same prompt)
- added metrics and answer_choices to prompts

* Fix choices format in prompt for craigslist_bargain.

* Clean health_fact.

* Clean hyperpartisan by article.

* Rename prompts for hyperpartisan by publisher.

Co-authored-by: Stephen Bach <stephenhbach@gmail.com>

Author: jason-fries

promptsource/templates/craigslist_bargains/templates.yaml

@@ -1,7 +1,7 @@
 dataset: craigslist_bargains
 templates:
   145dd841-b971-4550-bc88-305ad3278d58: !Template
-    answer_choices: null
+    answer_choices: good ||| bad ||| incomplete
     id: 145dd841-b971-4550-bc88-305ad3278d58
     jinja: 'The following conversation is a negotiation on craigslist. The first speaker
       is the buyer, and the second speaker is the seller.
@@ -33,32 +33,33 @@ templates:
 
       {% if final_price == -1 %}
 
-      incomplete
+      {{answer_choices[2]}}
 
       {% elif ((final_price - agent_info["Target"][0]) | abs) == ((final_price - agent_info["Target"][1])
       | abs) %}
 
-      good
+      {{answer_choices[0]}}
 
       {% elif ((final_price - agent_info["Target"][0]) | abs) < ((final_price - agent_info["Target"][1])
       | abs) %}
 
-      bad
+      {{answer_choices[1]}}
 
       {% elif ((final_price - agent_info["Target"][0]) | abs) > ((final_price - agent_info["Target"][1])
       | abs) %}
 
-      good
+      {{answer_choices[0]}}
 
       {% endif %}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
       original_task: false
     name: good deal for seller no list price implicit
     reference: implicit version of "good deal for seller no list price"
   27010b55-dd5b-4ee9-9e14-a4b809aa6cdb: !Template
-    answer_choices: null
+    answer_choices: yes ||| no ||| unknown
     id: 27010b55-dd5b-4ee9-9e14-a4b809aa6cdb
     jinja: 'The following conversation is a negotiation on craigslist. The first speaker
       is the buyer, and the second speaker is the seller.
@@ -67,7 +68,8 @@ templates:
       {{utterance | join("\n\n")}}
 
 
-      Was this a good deal for the seller? answer "yes", "no", or "unknown".
+      Was this a good deal for the seller? answer "{{answer_choices[0]}}", "{{answer_choices[1]}}",
+      or "{{answer_choices[2]}}".
 
       |||
 
@@ -90,27 +92,28 @@ templates:
 
       {% if final_price == -1 %}
 
-      unknown
+      {{answer_choices[2]}}
 
       {% elif ((final_price - agent_info["Target"][0]) | abs) == ((final_price - agent_info["Target"][1])
       | abs) %}
 
-      yes
+      {{answer_choices[0]}}
 
       {% elif ((final_price - agent_info["Target"][0]) | abs) < ((final_price - agent_info["Target"][1])
       | abs) %}
 
-      no
+      {{answer_choices[1]}}
 
       {% elif ((final_price - agent_info["Target"][0]) | abs) > ((final_price - agent_info["Target"][1])
       | abs) %}
 
-      yes
+      {{answer_choices[1]}}
 
       {% endif %}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
       original_task: false
     name: good deal for seller no list price
     reference: same as "good deal for seller" prompt, but excludes the list price
@@ -130,14 +133,15 @@ templates:
 
       ${{(agent_info[''Target''][0] - agent_info[''Target''][1]) | abs}}0'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Other
       original_task: false
     name: gap between parties
     reference: asks model explicitly what the gap was between the buyer's target and
       the seller's target
   78d1b487-c535-4a0d-ae49-055d321db3fd: !Template
-    answer_choices: null
+    answer_choices: yes ||| no ||| unknown
     id: 78d1b487-c535-4a0d-ae49-055d321db3fd
     jinja: 'The following conversation is a negotiation on craigslist. The first speaker
       is the buyer, and the second speaker is the seller. The listed price was ${{items["Price"][0]}}.
@@ -146,7 +150,8 @@ templates:
       {{utterance | join("\n\n")}}
 
 
-      Was this a good deal for the seller? Answer "yes" or "no", or "unknown".
+      Was this a good deal for the seller? Answer "{{answer_choices[0]}}" or "{{answer_choices[1]}}",
+      or "{{answer_choices[2]}}".
 
       |||
 
@@ -169,51 +174,51 @@ templates:
 
       {% if final_price == -1 %}
 
-      unknown
+      {{answer_choices[2]}}
 
       {% elif ((final_price - agent_info["Target"][0]) | abs) == ((final_price - agent_info["Target"][1])
       | abs) %}
 
-      yes
+      {{answer_choices[0]}}
 
       {% elif ((final_price - agent_info["Target"][0]) | abs) < ((final_price - agent_info["Target"][1])
       | abs) %}
 
-      no
+      {{answer_choices[1]}}
 
       {% elif ((final_price - agent_info["Target"][0]) | abs) > ((final_price - agent_info["Target"][1])
       | abs) %}
 
-      yes
+      {{answer_choices[0]}}
 
       {% endif %}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
       original_task: false
     name: good deal for seller
     reference: asks the model whether the deal was good for the seller or not (it's
       good if the seller's target is closer to the final price than the buyer's, or
       there is a tie)
   a1dbb258-2e5c-4160-986b-46fc03546965: !Template
-    answer_choices: null
+    answer_choices: buyer |||  seller ||| neither ||| unknown
     id: a1dbb258-2e5c-4160-986b-46fc03546965
     jinja: 'The following conversation is a negotiation on craigslist. The first speaker
       is the buyer, and the second speaker is the seller. The listed price was ${{items["Price"][0]}}.
 
 
       {{utterance | join("\n\n")}}
 
-
       Question: Which party got the better deal? Choose from:
 
-      a) the buyer
+      - {{answer_choices[0]}}
 
-      b) the seller
+      - {{answer_choices[1]}}
 
-      c) neither - it is a fair compromise
+      - {{answer_choices[2]}}
 
-      d) unknown
+      - {{answer_choices[3]}}
 
 
       Answer:
@@ -239,27 +244,28 @@ templates:
 
       {% if final_price == -1 %}
 
-      d) unknown
+      {{answer_choices[3]}}
 
       {% elif ((final_price - agent_info["Target"][0]) | abs) == ((final_price - agent_info["Target"][1])
       | abs) %}
 
-      c) neither - it is a fair compromise
+      {{answer_choices[2]}}
 
       {% elif ((final_price - agent_info["Target"][0]) | abs) < ((final_price - agent_info["Target"][1])
       | abs) %}
 
-      a) the buyer
+      {{answer_choices[0]}}
 
       {% elif ((final_price - agent_info["Target"][0]) | abs) > ((final_price - agent_info["Target"][1])
       | abs) %}
 
-      b) the seller
+      {{answer_choices[1]}}
 
       {% endif %}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
       original_task: false
     name: best deal
     reference: explicitly asks model which party got the best deal
@@ -269,11 +275,14 @@ templates:
     jinja: 'The following conversation is a negotiation on craigslist. The first speaker
       is the buyer, and the second speaker is the seller.
 
+
       {% set nonempty_utterance = [] %}
 
-      {% for line in utterance %}
+      {% for line in utterance  %}
+
+      {% if line != ""  %}
 
-      {% if line != "" %}{{ nonempty_utterance.append(line) or ""}}
+      {{ nonempty_utterance.append(line) or "" }}
 
       {% endif %}
 
@@ -291,8 +300,9 @@ templates:
 
       {{nonempty_utterance[-1]}}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Other
       original_task: true
     name: generate line
     reference: Generates the next line of negotiation

promptsource/templates/freebase_qa/templates.yaml

@@ -12,9 +12,10 @@ templates:
       \ %}\n{{answer.AnswersName[0][0].split(\" \") | map(\"capitalize\") | join(\"\
       \ \") }}"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: false
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      original_task: true
     name: qa_context_2
     reference: qa prompt with topic and inference chain provided
   1d583b71-7ef1-49df-b252-e8e1d6910129: !Template
@@ -33,8 +34,9 @@ templates:
 
       '
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
       original_task: false
     name: inference_chain_prompt
     reference: predicting the inference chain given just the question
@@ -45,8 +47,9 @@ templates:
       \ = Parses.Answers | choice %}\n{{answer.AnswersName[0][0].split(\" \") | map(\"\
       capitalize\") | join(\" \") }}"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
       original_task: true
     name: qa_template_basic
     reference: basic question/answer format
@@ -59,8 +62,9 @@ templates:
       \ \n{% set answer = Parses.Answers | choice %}\n{{answer.AnswersName[0][0].split(\"\
       \ \") | map(\"capitalize\") | join(\" \") }}"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
       original_task: true
     name: qa_context_1
     reference: qa question with simple entity context
@@ -73,8 +77,9 @@ templates:
       \ }}\"? \n||| \n{% set a = Parses.InferentialChain | first %}\n{{ a.split(\"\
       .\") | last | capitalize | replace(\"_\", \" \")}}\n"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
       original_task: false
     name: inference_chain_prompt_context
     reference: determine the inference chain between question and answer