Spaces:
Runtime error
Runtime error
Sebastian Gehrmann
commited on
Commit
·
ef1f338
1
Parent(s):
4081c39
added structured annotation section
Browse files- datacards/curation.py +71 -1
datacards/curation.py
CHANGED
@@ -12,7 +12,7 @@ from .streamlit_utils import (
|
|
12 |
|
13 |
N_FIELDS_ORIGINAL = 4
|
14 |
N_FIELDS_LANGUAGE = 12
|
15 |
-
N_FIELDS_ANNOTATIONS =
|
16 |
N_FIELDS_CONSENT = 0
|
17 |
N_FIELDS_PII = 0
|
18 |
N_FIELDS_MAINTENANCE = 0
|
@@ -148,11 +148,81 @@ def curation_page():
|
|
148 |
key_list=key_pref + ["filtered-criteria"],
|
149 |
help="Describe the process for selecting instances to include in the dataset, including any tools used. If no selection was done, enter N/A.",
|
150 |
)
|
|
|
151 |
with st.expander("Structured Annotations", expanded=False):
|
152 |
key_pref = ["curation", "annotations"]
|
153 |
st.session_state.card_dict["curation"][
|
154 |
"annotations"
|
155 |
] = st.session_state.card_dict["curation"].get("annotations", {})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
156 |
with st.expander("Consent", expanded=False):
|
157 |
key_pref = ["curation", "consent"]
|
158 |
st.session_state.card_dict["curation"]["consent"] = st.session_state.card_dict[
|
|
|
12 |
|
13 |
N_FIELDS_ORIGINAL = 4
|
14 |
N_FIELDS_LANGUAGE = 12
|
15 |
+
N_FIELDS_ANNOTATIONS = 10
|
16 |
N_FIELDS_CONSENT = 0
|
17 |
N_FIELDS_PII = 0
|
18 |
N_FIELDS_MAINTENANCE = 0
|
|
|
148 |
key_list=key_pref + ["filtered-criteria"],
|
149 |
help="Describe the process for selecting instances to include in the dataset, including any tools used. If no selection was done, enter N/A.",
|
150 |
)
|
151 |
+
|
152 |
with st.expander("Structured Annotations", expanded=False):
|
153 |
key_pref = ["curation", "annotations"]
|
154 |
st.session_state.card_dict["curation"][
|
155 |
"annotations"
|
156 |
] = st.session_state.card_dict["curation"].get("annotations", {})
|
157 |
+
|
158 |
+
make_radio(
|
159 |
+
label="Does the dataset have additional annotations for each instance?",
|
160 |
+
options=["none", "found", "automatically created", "expert created", "crowd-sourced"],
|
161 |
+
key_list=key_pref + ["origin"],
|
162 |
+
help="Was any additional data collected?",
|
163 |
+
)
|
164 |
+
|
165 |
+
# TODO: If yes....
|
166 |
+
# If expert or crowdsourced, this branch
|
167 |
+
make_radio(
|
168 |
+
label="What is the number of raters ",
|
169 |
+
options=["unknown", "1", "2<n<10", "11<n<50", "51<n<100", "n>100"],
|
170 |
+
key_list=key_pref + ["rater-number"],
|
171 |
+
help="How many raters were used to create the additional annotations?",
|
172 |
+
)
|
173 |
+
make_text_area(
|
174 |
+
label="Describe the qualifications required of an annotator.",
|
175 |
+
key_list=key_pref + ["rater-qualifications"],
|
176 |
+
help="e.g., languages or dialects they speak, education requirements, number of HITs (if MTurk).",
|
177 |
+
)
|
178 |
+
make_radio(
|
179 |
+
label="How many annotators saw each training example?",
|
180 |
+
options=["0", "1", "2", "3", "4", "5", ">5"],
|
181 |
+
key_list=key_pref + ["rater-training-num"],
|
182 |
+
help="",
|
183 |
+
)
|
184 |
+
make_radio(
|
185 |
+
label="How many annotators saw each test example?",
|
186 |
+
options=["0", "1", "2", "3", "4", "5", ">5"],
|
187 |
+
key_list=key_pref + ["rater-test-num"],
|
188 |
+
help="",
|
189 |
+
)
|
190 |
+
make_radio(
|
191 |
+
label="Was an annotation service used?",
|
192 |
+
options=["yes", "no", "unknown"],
|
193 |
+
key_list=key_pref + ["rater-annotation-service-bool"],
|
194 |
+
help="",
|
195 |
+
)
|
196 |
+
# TODO if yes
|
197 |
+
make_multiselect(
|
198 |
+
label="Which annotation services were used?",
|
199 |
+
options=[
|
200 |
+
"Amazon Mechanical Turk", "Prolific Academic",
|
201 |
+
"Upwork", "Appen", "Crowdflower", "other"
|
202 |
+
],
|
203 |
+
key_list=key_pref + ["rater-annotation-service"],
|
204 |
+
)
|
205 |
+
|
206 |
+
|
207 |
+
make_text_area(
|
208 |
+
label="Purpose and values for each annoation",
|
209 |
+
key_list=key_pref + ["values"],
|
210 |
+
help="Describe the purpose and possible values for each kind of annotation.",
|
211 |
+
)
|
212 |
+
make_multiselect(
|
213 |
+
label="Quality control measures?",
|
214 |
+
options=["none", "unknown", "validated by another rater", "validated by data curators", "validated through automated script", "other"],
|
215 |
+
key_list=key_pref + ["quality-control"],
|
216 |
+
help="How was annotation quality controlled for / what control measures were put in place to ensure annotation quality?",
|
217 |
+
)
|
218 |
+
# TODO: If not none / unknown
|
219 |
+
make_text_area(
|
220 |
+
label="Describe the quality control measures that were taken.",
|
221 |
+
key_list=key_pref + ["quality-control-details"],
|
222 |
+
help="Describe how quality was ensured in the data curation process.",
|
223 |
+
)
|
224 |
+
|
225 |
+
|
226 |
with st.expander("Consent", expanded=False):
|
227 |
key_pref = ["curation", "consent"]
|
228 |
st.session_state.card_dict["curation"]["consent"] = st.session_state.card_dict[
|