@inproceedings{BS_RE14,
 author = {Travis D. Breaux and Florian Schaub},
 affiliation = {Carnegie Mellon University},
 title = {Scaling Requirements Extraction to the Crowd},
 year = {2014},
 month = {August},
 booktitle = {RE'14: Proceedings of the 22nd IEEE International Requirements Engineering Conference (RE'14)},
 publisher = {IEEE Society Press},
 address = {Washington, DC, USA},
 pages = {},
 location = {Karlskrona, Sweden},
 abstract = {Natural language text sources have increasingly been used to develop new methods and tools for extracting and analyzing requirements. To validate these new approaches, researchers rely on a small number of trained experts to perform a labor-intensive manual analysis of the text. The acquired data is then used to develop a reliable gold standard. The time and resources needed to conduct manual extraction, however, has limited the size of case studies and thus the generalizability of results. To begin to address this problem, we conducted three experiments to evaluate crowdsourcing a manual requirements extraction task from text sources describing data collection, sharing and usage requirements. In these studies, we carefully balance worker payment and overall cost, as well as worker training and data quality to study the feasibility of distributing requirements extraction to the crowd. We present results from two pilot studies and third experiment to justify a task decomposition approach to requirements extraction. Our contributions include the task decomposition workflow and three metrics for measuring worker performance. The final evaluation shows a 60% reduction in the cost of manual extraction with a 16% increase in extraction coverage of for data actions.},
}