@conference {
	title = {Crowdsourcing with Multi-Dimensional Trust and Active Learning},
	author = {Liu, Xiangyang., Baras, John S.},
	year = {2017},
	month = {12/2017},
	abstract = {We consider a typical crowdsourcing task that aggregates input from multiple workers as a problem in information fusion. To cope with the issue of noisy and sometimes malicious input from users, trust is used to model workers expertise. We propose a probabilistic model to jointly infer multi-dimensional trust of workers, multi-domain properties of questions,and true labels of questions.Our model is flexible and extensible to incorporate metadata associated with questions. To show that, we further propose two extended models, one of which handles input tasks with real-valued features and the other handles tasks with text features by incorporating topic models. In order to decrease entropies and reduce error rates more quickly with fewer annotations from workers, we further propose strategies for selecting which questions to ask and which workers to assign the questions to based on multidimension characteristics of questions and workers trust values in those dimensions. We evaluate our models and algorithms on real-world data sets. These results can be applied for fusion of information from multiple data sources like sensors, human input, machine learning results, or a hybrid of them.~},
	address = {Melbourne, Australia},
	booktitle = {56th IEEE Conference on Decision and Control (CDC 2017)},
	pages = {pp. 3224-3231}
}
@conference {
	title = {Trust-Aware Crowdsourcing with Domain Knowledge},
	author = {Liu, Xiangyang., Baras, John S.},
	year = {2015},
	month = {12/2015},
	abstract = {The rise of social network and crowdsourcing platforms~makes it convenient to take advantage of the collective~intelligence to estimate true labels of questions of interest.~However, input from workers is often noisy and even malicious.~Trust is used to model workers in order to better~estimate true labels of questions. We observe that questions~are often not independent in real life applications. Instead,~there are logical relations between them. Similarly, workers~that provide answers are not independent of each other~either. Answers given by workers with similar attributes~tend to be correlated. Therefore, we propose a novel unified~graphical model consisting of two layers. The top layer~encodes domain knowledge which allows users to express~logical relations using first-order logic rules and the bottom~layer encodes a traditional crowdsourcing graphical model.~Our model can be seen as a generalized probabilistic soft~logic framework that encodes both logical relations and~probabilistic dependencies. To solve the collective inference~problem efficiently, we have devised a scalable joint inference~algorithm based on the alternating direction method~of multipliers. Finally, we demonstrate that our model is~superior to state-of-the-art by testing it on multiple real-world~datasets.},
	address = {Osaka, Japan},
	booktitle = {54th Conference on Control (CDC 2015)},
	pages = {pp. 2913-2918}
}
@conference {
	title = {Crowdsourcing with Multi-Dimensional Trust},
	author = {Liu, Xiangyang., He, He., Baras, John S.},
	year = {2015},
	month = {07/2015},
	abstract = {We consider a typical crowdsourcing task that aggregates input from multiple workers as a problem in information fusion. To cope with the issue of noisy and sometimes malicious input from workers, trust is used to model workers{\textquoteright} expertise. In a multi-domain knowledge learning task, however, using scalar-valued trust to model a worker{\textquoteright}s performance is not sufficient to reflect the worker{\textquoteright}s trustworthiness in each of the domains. To address this issue, we propose a probabilistic model to jointly infer multi-dimensional trust of workers, multi-domain properties of questions, and true labels of questions. Our model is very flexible and extensible to incorporate metadata associated with questions. To show that, we further propose two extended models, one of which handles input tasks with real-valued features and the other handles tasks with text features by incorporating topic models. Finally, we evaluate our model on real-world datasets and demonstrate that our model is superior to state-of-the-art and the two extended models have even better performance. In addition, our models can effectively recover trust vectors of workers, which can be very useful in task assignment adaptive to workers{\textquoteright} trust in the future. These results can be applied for fusion of information from multiple data sources like sensors, human input, machine learning results, or a hybrid of them.},
	address = {Washington DC},
	booktitle = {International Conference on Information Fusion (FUSION 2015)},
	pages = {pp. 574-581}
}
@conference {
	title = {Trust-Aware Optimal Crowdsourcing With Budget Contraint},
	author = {Liu, Xiangyang., He, He., Baras, John S.},
	year = {2015},
	month = {06/2015},
	abstract = {Crowdsourcing has been extensively used for aggregating data from a large pool of workers. In a real crowdsourcing market, each answer obtained from a worker incurs cost. The cost is associated with both the level of trustworthiness of workers and the difficulty of tasks. Typically, access to expert-level (more trustworthy) workers is more expensive than to average crowd and completion of a challenging task is more costly than a click-away question. In this paper, we address the problem of optimal assignment of heterogeneous tasks to workers of varying trust levels with budget constraint. Specifically, we design a trust-aware task allocation algorithm that takes as inputs the estimated trust of workers and pre-set budget and outputs the optimal assignment of tasks to workers. We derive the bound of total error probability that relates to budget, trustworthiness of crowds, and costs of obtaining labels from crowds naturally. Higher budget, more trustworthy crowds, and less costly jobs result in lower theoretical bound. Our allocation scheme does not depend on the specific design of the trust evaluation component. Therefore, it can be combined with generic trust evaluation algorithms. Our algorithm outperforms state-of-the-art by up to 30\% on real data.},
	address = {London, UK},
	booktitle = {IEEE International Conference on Communications (ICC 2015)},
	pages = {pp. 1176-1181}
}
@conference {
	title = {Using Trust in Distributed Consensus with Adversaries in Sensor and Other Networks},
	author = {Liu, Xiangyang., Baras, John S.},
	year = {2014},
	month = {07/2014},
	abstract = {Extensive research efforts have been devoted to distributed consensus with adversaries. Many diverse applications drive this increased interest in this area including distributed collaborative sensor networks, sensor fusion and distributed collaborative control. We consider the problem of detecting Byzantine adversaries in a network of agents with the goal of reaching consensus. We propose a novel trust model that establishes both local trust based on local evidences and global trust based on local exchange of local trust values. We describe a trust-aware consensus algorithm that integrates the trust evaluation mechanism into the traditional consensus algorithm and propose various local decision rules based on local evidence. To further enhance the robustness of trust evaluation itself, we also provide a trust propagation scheme in order to take into account evidences of other nodes in the network. Then we show by simulation that the trust-aware consensus algorithm can effectively detect Byzantine adversaries and excluding them from consensus iterations even in sparse networks with connectivity less than 2f + 1, where f is the number of adversaries. These results can be applied for fusion of trust evidences as well as for sensor fusion when malicious sensors are present like for example in power grid sensing and monitoring.},
	address = {Salamanca, Spain},
	booktitle = {17th International Conference on Information Fusion (Fusion 2014)}
}
@conference {
	title = {Distributed Consensus with Byzantine Adversaries},
	author = {Liu, Xiangyang., Gao, Peixin., Baras, John S.},
	year = {2013},
	month = {10/2013},
	abstract = {The problem of distributed consensus algorithms in the presence of Byzantine adversaries has been increasingly important in both distributed computing scenaria and cyber-physical systems. Yet a satisfactory solution to the problem is still being sought due to the fact that Byzantine adversaries can output arbitrary wrong states and even collude to cause worse damages. We aim to solve the problem via detecting adversaries. Related works in multi-party computation in computer science are addressing this type of problem relying on the assumption that adversaries cannot lie about their input values. Works in the control theoretic literature rely on strict and hardly verifiable network robustness conditions. Both lines of research just consider guaranteeing the consensus in the face of adversaries yet neither of them considers detecting adversaries. In this paper a method is proposed to investigate this problem based on the detection of adversaries in the execution of the consensus algorithm. We integrate a trust evaluation mechanism into the consensus algorithm, where trust can be established with the help of a special subset of nodes, namely headers. We propose a two-layer hierarchical framework, where the top layer is a super-step dealing with a vectorized consensus algorithm, and the bottom layer is a sub-step, executing our proposed parallel vectorized voting scheme. The parallel vectorized voting scheme uses the information received by the top layer as an initial trust opinion and then evolves to equilibrium trust opinions. The vectorized consensus algorithm in the top layer utilizes the trust evaluation results provided by the bottom layer to update confidence levels and states. We also derive an upper bound of the number of adversaries that our algorithm can resist in each super-step.},
	address = {Montecello, Illinois},
	booktitle = {Proceedings of 51st Annual Allerton Conference on Communication, Control and Computing}
}
@article {
	title = {Crowdsourcing with Multi Dimensional Trust and Active Learning},
	author = {Liu, Xiangyang., He He, Baras, John S.},
	year = {2016},
	month = {06/2016},
	abstract = {We consider a typical crowdsourcing task that aggregates input from multiple workers as a problem in information fusion. To cope with the issue of noisy and sometimes malicious input from workers, trust is used to model workers expertise. In a multi-domain knowledge learning task, however, using scalarvalued trust to model a workers performance is not sufficient to reflect the workers trustworthiness in each of the domains. To address this issue, we propose a probabilistic model to jointly infer multi-dimensional trust of workers, multi-domain properties of questions, and true labels of questions. Our model is flexible and extensible to incorporate metadata associated with questions. To show that, we further propose two extended models, one of which handles input tasks with real-valued features and the other handles tasks with text features by incorporating topic models. Finally, we evaluate our model on real-world datasets and demonstrate that our models are superior to state-of-the-art. In addition, our models can effectively recover trust vectors of workers, which can be very useful in task assignment adaptive to workers trust in the future. In order to decrease entropies and reduce error rates more quickly with fewer annotations from workers, we further propose strategies for selecting which questions to ask and which workers to assign the questions to based on multi- dimension characteristics of questions and workers trust values in those dimensions. These results can be applied for fusion of information from multiple data sources like sensors, human input, machine learning results, or a hybrid of them.},
	journal = {Journal of Advances in Information Fusion}
}
@article {
	title = {Crowdsourcing with Multi Dimensial Trust and Active Learning},
	author = {Liu, Xiangyang., He, He., Baras, John S.},
	year = {2016},
	month = {01/2016},
	abstract = {We consider a typical crowdsourcing task that aggregates input from multiple workers as a problem in information fusion. To cope with the issue of noisy and sometimes malicious input from workers, trust is used to model workers expertise. In a multi-domain knowledge learning task, however, using scalar-valued trust to model a workers performance is not sufficient to reflect the worker{\textquoteright}s trustworthiness in each of the domains. To address this issue, we propose a probabilistic model to jointly infer multi-dimensional trust of workers, multi-domain properties of questions, and true labels of questions. Our model is flexible and extensible to incorporate metadata associated with questions. To show that, we further propose two extended models, one of which handles input tasks with real-valued features and the other handles tasks with text features by incorporating topic models. Finally, we evaluate our model on real-world datasets and demonstrate that our models are superior to state-of-the-art. In addition, our models can effectively recover trust vectors of workers, which can be very useful in task assignment adaptive to workers trust in the future. In order to decrease entropies and reduce error rates more quickly with fewer annotations from workers, we further propose strategies for selecting which questions to ask and which workers to assign the questions to based on multi- dimension characteristics of questions and workers trust values in those dimensions. These results can be applied for fusion of information from multiple data sources like sensors, human input, machine learning results, or a hybrid of them.~},
	journal = {Journal of Advances in Information Fusion}
}
@article {
	title = {Trust is the Cure to Distributed Consensus with Adversaries},
	author = {Liu, Xiangyang., Baras, John S.},
	year = {2016},
	month = {01/2016},
	abstract = {Extensive research efforts have been devoted to distributed~consensus with adversaries. Many diverse applications~drive this increased interest in this area including~distributed collaborative sensor networks, sensor fusion~and distributed collaborative control. We consider the~problem of detecting Byzantine adversaries in a network~of agents with the goal of reaching consensus. We~propose a novel trust model that establishes both local~trust based on local evidences and global trust based~on local exchange of local trust values. We describe~a trust-aware consensus algorithm that integrates the~trust evaluation mechanism into the traditional consensus~algorithm and propose various local decision rules based~on local evidence. To further enhance the robustness of~trust evaluation itself, we also provide a trust propagation~scheme in order to take into account~evidences of other~nodes in the network. The algorithm is flexible and extensible~to incorporate more complicated designs of decision~rules and trust models. To demonstrate the power of~our trust propagation scheme, we provide theoretical~security performance in terms of miss detection rate and~false alarm rate under regular trust graph and relaxed~security performance bound under general trust graph.~In addition, we demonstrate through simulation that the~trust-aware consensus algorithm can effectively detect~Byzantine adversaries and excluding them from consensus~iterations even in sparse networks with connectivity~less than 2f + 1, where f is the number of adversaries.~These results can be applied for fusion of trust evidences~as well as for sensor fusion when malicious sensors~are present like for example in power grid sensing~and monitoring.~},
	journal = {IEEE Transactions on Automatic Control}
}
