by Umair ul Hassan, Edward Curry
Abstract:
Assigning heterogeneous tasks to workers is an important challenge of crowdsourcing platforms. Current ap- proaches to task assignment have primarily focused on content- based approaches, qualifications, or work history.We propose an alternative and complementary approach that focuses on what capabilities workers employ to perform tasks. First, we model various tasks according to the human capabilities required to perform them. Second, we capture the capability traces of the crowd workers performance on existing tasks. Third, we predict performance of workers on new tasks to make task routing decisions, with the help of capability traces. We evaluate the ef- fectiveness of our approach on three different tasks including fact verification, image comparison, and information extraction. The results demonstrate that we can predict worker's performance based on worker capabilities. We also highlight limitations and extensions of the proposed approach. Keywords—microtask,
Reference:
Umair ul Hassan, Edward Curry, "A Capability Requirements Approach for Predicting Worker Performance in Crowdsourcing", In Proceedings of the 9th IEEE International Conference on Collaborative Computing: Networking, Applications and Worksharing, ICST, Austin, Texas, pp. 1-9, 2013. 25% acceptance rate
Bibtex Entry:
@inproceedings{UlHassan2013a,
abstract = {Assigning heterogeneous tasks to workers is an important challenge of crowdsourcing platforms. Current ap- proaches to task assignment have primarily focused on content- based approaches, qualifications, or work history.We propose an alternative and complementary approach that focuses on what capabilities workers employ to perform tasks. First, we model various tasks according to the human capabilities required to perform them. Second, we capture the capability traces of the crowd workers performance on existing tasks. Third, we predict performance of workers on new tasks to make task routing decisions, with the help of capability traces. We evaluate the ef- fectiveness of our approach on three different tasks including fact verification, image comparison, and information extraction. The results demonstrate that we can predict worker's performance based on worker capabilities. We also highlight limitations and extensions of the proposed approach. Keywords—microtask,},
address = {Austin, Texas},
annote = {25% acceptance rate},
author = {ul Hassan, Umair and Curry, Edward},
booktitle = {Proceedings of the 9th IEEE International Conference on Collaborative Computing: Networking, Applications and Worksharing},
doi = {10.4108/icst.collaboratecom.2013.254181},
file = {:Users/ed/Library/Application Support/Mendeley Desktop/Downloaded/ul Hassan, Curry - 2013 - A Capability Requirements Approach for Predicting Worker Performance in Crowdsourcing.pdf:pdf},
isbn = {978-1-936968-92-3},
keywords = {crowdsourcing,microtask,performance,performancea,taxonomy},
mendeley-tags = {crowdsourcing,microtask,performance,taxonomy},
pages = {1--9},
publisher = {ICST},
title = {{A Capability Requirements Approach for Predicting Worker Performance in Crowdsourcing}},
url = {http://www.edwardcurry.org/publications/hassan_collabcom13.pdf},
year = {2013}
}