\documentclass{bioinfo}
\usepackage{booktabs}
\usepackage[export]{adjustbox}
\copyrightyear{2005}
\pubyear{2005}

\begin{document}
\firstpage{1}
\setlength\abovedisplayskip{4pt}
 \setlength\belowdisplayskip{4pt}
\setlength{\itemsep}{0pt}
\setlength{\topsep}{0pt}

\newcommand{\ra}[1]{\renewcommand{\arraystretch}{#1}}



\title[Parametric Bayesian Priors and Better Choice of Negative Examples]{Parametric Bayesian Priors and Better Choice of Negative Examples Improve Protein Function Prediction}
\author[N. Youngs \textit{et~al}]{Noah Youngs\,$^{1}$, Duncan Penfold-Brown\,$^{2}$,  Kevin Drew\,$^{2}$, Dennis Shasha\,$^{1,*}$, and Richard Bonneau\,$^{1,2}$
\footnote{to whom correspondence should be addressed}}
\address{$^{1}$Department of Computer Science, New York University, New York, USA\\
$^{2}$Center for Genomics and Systems Biology, Department of Biology, New York University, New York, USA\\}

\history{Received on XXXXX; revised on XXXXX; accepted on XXXXX}

\editor{Associate Editor: XXXXXXX}

\maketitle

\begin{abstract}

\section{Motivation:}
Computational biologists have demonstrated the utility of using machine learning methods to predict protein function from an integration of multiple genome-wide data types. Yet even the best performing function prediction algorithms rely on heuristics for important components of the algorithm, such as choosing negative examples (proteins without a given function), or determining key parameters. The improper choice of negative examples in particular can hamper the accuracy of protein function prediction.

\section{Results:}
We present a novel approach for choosing negative examples, using a parametrizable Bayesian prior computed from all observed annotation data, which also generates priors used during function prediction.  We incorporate this new method into the GeneMANIA function prediction algorithm and demonstrate improved accuracy of our algorithm over current top-performing function prediction methods on the yeast and mouse proteomes across all metrics tested.

\section{Availability:}
 Code and Data is available at:
\\
http://bonneaulab.bio.nyu.edu/funcprop.html

\section{Contact:} \href{shasha@courant.nyu.edu, bonneau@cs.nyu.edu}{shasha@courant.nyu.edu, bonneau@cs.nyu.edu}
\end{abstract}

\section{Introduction}
The rate of new protein discovery has, in recent years, outpaced our ability to annotate and characterize new proteins and proteomes. In order to combat this functional annotation deficit, many groups have successfully turned to computational techniques, attempting to predict the function of proteins in order to guide experimental verification. Specifically, there has been a surge of interest in applying traditional machine learning methods to the problem of protein function prediction (FP), in order to take advantage of the the wealth of biological data available for each protein beyond its sequence, such as computationally-predicted tertiary structure which has already been shown to aid FP (Drew \textit{et al.}, 2011). While traditional approaches to FP mainly involved either homology (with limitations of accuracy) or manual curation (dependent on rare expertise), these new and more complex methods present new evaluation and comparative challenges. The MouseFunc competition (Pena-Castillo \textit{et al.}, 2008) was organized to showcase the ability of machine learning methods to take advantage of large integrated data-sets and provide useful predictions of gene function.
\\
	\indent The validity of integrative approaches to function prediction was first demonstrated by the works of Marcotte \textit{et al.} (1999) and Troyanskaya \textit{et al.} (2003), which respectively used linkage graphs and a Bayesian network to predict function. By the time of the MouseFunc competition, FP methods had become quite diverse, including: Support Vector Machines, Random Forests, Decision Trees, and several composite methods (Guan \textit{et al.}, \negthinspace 2008; Lee \textit{et al.}, \negthinspace 2006; Obozinski \textit{et al.}, \negthinspace 2008; Tasan \textit{et al.}, \negthinspace 2008), but a recurring theme was to use protein-protein networks of various types to determine function based on guilt-by-association (Kim \textit{et al.}, \negthinspace 2008; Leone \textit{et al.}, \negthinspace 2005; Qi \textit{et al.}, \negthinspace 2008; Zhang \textit{et al.}, \negthinspace 2008). In such a method, genes are represented by nodes in a network, with weighted edges defined by a similarity metric obtained from raw data (often the Pearson Correlation Coefficient of feature vectors). Predictions are then formed by propagating information from genes known to have a function, through the network to unlabeled genes.
\\
	\indent While providing unprecedented accuracy, the methods of the MouseFunc competition exposed several general challenges still remaining for the FP problem: 1) choosing a set of high-confidence negative examples, 2) utilizing available data to form prior beliefs about the biological functions of a gene, and 3) effectively combining disparate data sources. As no comprehensive database of functional negative examples currently exists, and nearly all major machine learning methods require a negative class for the training of a classifier, the selection of high-confidence negative examples is especially important for the FP problem.
	\\
	\indent In this work, we begin to address these challenges by presenting a parameterizable Bayesian technique for computing prior functional biases for each gene, and a novel method for selecting negative examples using these biases. To apply our method, we utilize the framework of the GeneMANIA algorithm (Mostafavi \textit{et al.}, \negthinspace 2008), one of the highest-performing competitors in MouseFunc. In addition to our new priors and negative examples, we present a framework for tuning our Bayesian parameters and other parameters in the original GeneMANIA algorithm. To facilitate this parameter tuning, we incorporate new optimization techniques that take advantage of the structure of the optimization problem. We also integrate our novel negative examples into the GeneMANIA network combination algorithm that synthesizes heterogeneous data into one affinity network.
\\
	\indent While well-established procedures exist for the comparison of machine learning methods, recent work (Greene and Troyanskaya, 2012; Pavlidis and Gillis, 2012) has exposed and discussed problems that can be introduced into these comparisons by the nature of biological data. In order to mitigate these biases, we heed the suggestions of Greene and Troyanskaya (2012), and focus on evaluation with a temporal hold out (referred to in this paper and in the MouseFunc competition as the "novel evaluation setting"). We also include one of the few available gold standard evaluation sets, obtained from Huttenhower \textit{et al.} (2009). It is important to keep in mind when examining our evaluation results, that our goal is to demonstrate the performance improvements of our new algorithm over the existing state-of-the-art in a fair comparison across several datasets, rather than to make any generalized claim about performance on all function prediction tasks.

\section{Previous Work}
We present our novel methods in the context of the GeneMANIA function prediction algorithm of Mostafavi \textit{et al.} (2008), which incorporates prior beliefs and an intelligent network combination algorithm into its guilt-by-association framework. GeneMANIA is a form of Gaussian Random Field (GRF) label propagation, a semi-supervised technique pioneered by Zhou \textit{et al.} (2004); Zhu \textit{et al.} (2003), and provides predictions for genes one function at a time. Given a set of nodes (genes) in a network whose edges define pairwise similarity, and a vector $\vec{y}$ of prior label biases for the nodes given the current function being examined, the GRF algorithm assigns a discriminant value $f_i$ to each node, which can be ranked in order to produce predictions. The label biases $y_i$ take values in [-1,1], with -1 representing known negative labels, 1 representing known positive labels, and values in between reflecting prior belief about the likelihood of a gene having the function in question. The final discriminant vector $\vec{f}$ is obtained by solving the optimization problem:
\begin{equation}
\min_f(\sum (f_i- y_i)^2 + \sum\sum W_{ij}(f_i - f_j)^2) \label{eq:01}
\end{equation}	
This equation has an analytical solution in the form of a linear system: $A x\negmedspace =\negmedspace b$ (see Supplementary Materials for details), and also guarantees that the discriminant values $f_i$ will lie in the range [-1,1], with larger values indicating greater likelihood of an unlabeled node being a positive example of the function in question.
\\
	\indent Intuitively, this algorithm allows prior information to flow through the network until equilibrium is reached. The objective function propagates known labels through the similarity network via the second ``smoothness'' term in Equation~(1), weighted by the strength of similarity between nodes as specified by the network, and also enforces adherence to the prior bias through the first ``consistency'' term in Equation~(1). Thus the label biases, both the positive and negative examples as well as biases used for unlabeled nodes, play a very important role in the algorithm. Mostafavi and Morris (2009) explore variations on techniques to choose the label bias vector, but we expand upon this work to improve accuracy in our algorithm by utilizing more of the information contained in current functional annotations to determine functional biases and negative examples (see section 3.1).
\\
	\indent The other key component of the GRF algorithm is the composite network defining similarity between all pairs of genes. Mostafavi \textit{et al.} (2008) proposed a method to combine disparate data sources, each represented as an affinity matrix, into one composite matrix, based on the work of Tsuda \textit{et al.} (2005). This algorithm, for each GO category of interest, maximizes the similarity between pairs of positively labeled genes and minimizes the similarity between genes of opposite labels (see Supplementary Materials for details). 
	\indent This network combination algorithm is prone to over-fitting in cases with few positive examples. The original GeneMANIA algorithm addressed this problem by introducing a regularization term, but later work (Mostafavi and Morris, \negthinspace 2010) instead attempts to fit the composite data network for multiple GO categories simultaneously. Our algorithm expands upon this second approach by directly incorporating our negative examples (see section 3.3).

\begin{methods}

\section{Algorithm}
We propose novel techniques focusing on several key aspects of protein function prediction: choosing negative examples, forming label biases for unlabeled genes with some known annotations, and an issue specific to GRF-based methods, namely combining heterogeneous data-types into one affinity network. In addition we suggest a new optimization algorithm tailored to our techniques, and provide a framework for tuning parameters using the training data.


\begin{figure*}[!htb]
\includegraphics[max size={\textwidth}{\textheight}]{exampleFigure.eps}
\caption{(a) A subsection of the association network before the algorithm is run, showing prior beliefs for genes for the function: GO:0008194, UDP-glycotransferase activity, focusing on gene "Ogt". The shading of the nodes represents the degree of positivity compared to the mean of all prior biases, with blue indicating greater likelihood of possessing the function in question, red lesser likelihood, and white representing genes that had no GO annotations to use for a prior. Square nodes represent validation true positives (including the training positive example "Wdfy3"). (b) The same subsection of the association network as (a), but after label propagation, showing the final discriminant values of the genes. (c) The  GO categories that are most predictive of the function GO:0008194, with darker shades of blue representing stronger predictors.}\label{fig:01}
\end{figure*}

\subsection{Label Biases}
Mostafavi and Morris (2009) showed that significant performance gain could be achieved by allowing existing GO annotations to inform the priors applied to genes in GRF function prediction, using a technique called Hierarchical Label Bias (HLBias). This idea is supported by the work of King \textit{et al.} (2003), which showed that patterns of GO annotations alone provided enough signal to predict future annotations. HLBias specified that genes which possessed annotations for functions ancestral to the function of interest received a prior bias equal to the proportion of genes with the ancestral function that also are known to have the function in question.
\\
\indent However, due to the difficulty of defining a functional hierarchy, the structure of the GO tree is often altered by its curators, with terms being moved to different parents, virtually guaranteeing that there exist functional relationships that are non-ancestral. When considering the complexity of functional interactions, it would seem likely that the presence of some functions might influence the likelihood of a gene possessing other functions, regardless of whether or not the relationship between the two is ancestral. This is especially true when considering annotations in all three branches of the GO hierarchy simultaneously. Accordingly, we extend HLBias to include the likelihood of a given function co-occurring with all other existing annotations (across all three branches of the GO Tree: Biological Process, Molecular Function, and Cellular Component), in the following manner:
\\
	\indent Let $\hat{p}(c|m)$ denote the empirical conditional probability of seeing annotation $c$ given the presence of annotation $m$: $\hat{p}(c|m) = \frac{n_{mc}^+}{n_m^+}$
Where $n_{mc}^+$  is the number of gene products where both $m$ and $c$ appear and $n_m^+$  is the number of gene products that have annotation $m$. For a protein $i$, let $D_i$ be the set of all GO terms annotated to $i$. We approximate the conditional prior probability of gene $i$ having function $c$ by the score:
\begin{equation}
prior_i = \frac{1}{|D_i|} \sum_{m \in D_i} \hat{p}(c|m) \label{eq:04}
\end{equation}	
The label biases are then scaled to the range $[-1,1]$: $y_i = 2*prior_i - 1$.
\\
	\indent Due to the hierarchical nature of GO categories, some of the conditional probabilities in this calculation will contain redundant information, and so when considering a protein with annotations $D_i$, we remove from $D_i$ all GO terms which have a child in $D_i$, leaving a set of only the most specific annotations of protein $i$ to use in calculating the bias.
\\
\indent Figure 1.c provides an example of the most predictive GO terms for the GO function UDP-glycotransferase activity (UDPGA), which include many terms which have no ancestral relationship to UDPGA. Examining a specific prediction example, we find that the annotations informing the prior bias for gene Ogt (pictured in Figure 1.a), are all non-ancestral terms, and contribute to the algorithm making a correct positive prediction (Figure 1.b).
\\
	\indent Lastly, we observe a large bias introduced by categories with small sample size, where one category appears to be a perfect predictor of another. In order to reduce the potential for overfitting stemming from this phenomenon, we introduce a weighted pseudocount into the calculation of the empirical conditional probability, whereby $\hat{p}(c|m) = \frac{n_{mc}^+}{n_m^+}$ is replaced by:
\begin{equation}
\hat{p'}(c|m) = \frac{n_{mc}^+}{n_m^+ + \gamma e^{\lambda n_m^+}}
\end{equation}
This idea is motivated by the hypotheses that no GO category co-occurs with any non-ancestral category in every true case, and that the number of remaining but currently unknown occurrences of a function is related to the number of currently known occurrences. This equation (via the two parameters $\gamma$ and $\lambda$) allows us to smoothly transition between two extreme assumptions about how missing and currently known annotations are distributed: 1)  the number of observations in the data is a proxy for how well a function has been studied, and so the number of missing counts in the data should be inversely proportional to the number already seen, and 2) the number of currently known occurrences is in fact a better representation of the specificity of a function, and so the undiscovered occurrences should be directly proportional to the number already seen.
\\
	\indent In order to allow the data itself to choose one of these hypotheses, we sample from a range of combinations of parameters, including the potential for no pseudocounting (see Supplementary Materials). The final value of the parameters is chosen by tuning with cross-validation over the training set, as described in section 3.5.
\\
	\indent For genes with no previous annotations in GO, we follow Mostafavi \textit{et al.} (2008) and set the label bias to the mean of all the label biases calculated for genes with GO annotations, including the positive and negative example genes with values of $\{1,-1\}$ respectively. We refer to our label bias algorithm hereafter as ALBias.

\subsection{Negative Examples}
The choice of negative training examples for use in supervised machine learning algorithms is a recurring problem for FP methods. While the GO database does include negative annotations, the number of such annotations is currently small. Thus it is necessary to infer negative examples for each function (typically using a heuristic). Past heuristics include (i) designating all genes that don't have a particular label as being negative for that label (Guan \textit{et al.}, \negthinspace 2008), (ii) randomly sampling genes and assuming the probability of getting a false negative is low (often done when predicting protein-protein interactions, as in Gomez \textit{et al.}, 2003), and (iii) using genes with annotations in sibling categories of the category of interest as negative examples (Mostafavi and Morris, \negthinspace 2009). Mostafavi and Morris (2009) note in discussion that this last technique may often break down as some genes are annotated to more than one sibling category, and many genes have few siblings to use.
\\
	\indent We present a new technique for choosing negative examples based upon the label biases calculated for each function. Namely, all genes with an annotation in the same branch of GO as the term being predicted, and which have a $prior_i$ score of 0 for the function in question (with the prior score computed across all three branches of GO), are treated as negative examples for that function. Intuitively, this amounts to treating genes as negative if their most specific annotations have never appeared alongside the annotation in question in any gene. (Note that the choice of pseudocounting parameters does not impact the negative examples, as only the magnitude of the label bias will be affected and not whether a bias is non-zero.) 
\\
	\indent Restricting the negative examples to having an annotation in the same branch as the GO term being predicted, rather than simply having an annotation in any branch, does decrease the number of negative examples, but also more significantly decreases the number of validation true positives that were misclassified as negatives. The number of negatives in mouse decreased by 14.9\% due to this restriction, while the number of verifiable misclassified negatives dropped by 22.5\%, and in yeast the number of negative examples decreased by 23.2\% while the negative verifiable misclassified negatives dropped by 91.5\%.

\subsection{Network Weighting}
As mentioned in our description of previous work, one essential component of the GRF algorithm is synthesizing heterogeneous data sources into one pairwise affinity matrix. Mostafavi and Morris (2010) found that fitting this matrix for multiple GO functions simultaneously significantly decreased overfitting, especially in low-annotation categories. The authors simplified the calculation of this simultaneous fit by considering negative-negative pairs of labels as well as the positive-positive and positive-negative pairs utilized by the original network weighting algorithm of Mostafavi \textit{et al.} (2008). This simplification also requires the treatment of all non-positive genes as negative genes for each GO category (see Supplementary Materials for details).
\\
	\indent Mostafavi and Morris (2010) showed that these simplifications do not hamper performance, and also found that fitting the combined network to all GO categories in a particular branch (GO-BP, GO-CC, or GO-MF) worked better than any other subset or grouping of functions. We concur that fitting to all categories performs better than any of the subsets we attempted, but propose that the apparent indifference of this algorithm to the assumption that all non-positive nodes are negative was most likely due to a lack of any satisfactory alternative for choosing negative examples.
\\
	\indent We return to the unsimplified version of the simultaneous fit proposed by Mostafavi and Morris (2010), and utilize our more-specific negative examples that are unique to each GO category (see Supplementary Materials for details of the calculation). We refer to our modified network combination algorithm as Simultaneous Weights with Specific Negatives (SWSN).

\subsection{Successive Block Conjugate Gradient Optimization}
The network-weighting scheme defined above creates a single combined matrix W for all functional categories within the same GO branch. Thus the coefficient matrix is identical for the optimization problem that is solved for each function, and so we are faced only with the issue of a different right-hand-side (RHS) per function. In such cases, computational costs can be decreased by methods that solve all of the problems simultaneously, rather than iteratively solving each problem without using any of the information obtained by other solutions. We propose a modified version of the Successive Block Conjugate Gradient algorithm (SBCG) of Suarjana and Law (1994). 
\\
	\indent In this algorithm, the search direction is obtained simultaneously for all of the distinct RHS vectors in the problem. If at any point the search direction matrix becomes rank-deficient, dependent RHS vectors are moved to a secondary system, but are still updated with steps obtained from the search direction in the primary system, and so still proceed towards convergence. The speed of this secondary convergence is dependent on the angle between the vectors in the primary system and secondary system.
\\
	\indent Our algorithm differs from the original proposed by Suarjana and Law (1994) in several ways. Firstly, not all solutions converge to the desired tolerance in the same number of iterations, and so we save computation by removing already-converged RHS vectors from the block calculation rather than updating the entire system until all RHS vectors converge. Secondly, when the RHS vectors in the secondary system are nearly orthogonal to those in the primary system, waiting for secondary convergence can require a large number of iterations. Instead, once all primary system RHS vectors are converged, we restart the algorithm in a second phase with the secondary system as the primary system, but using the latest residuals as our starting point. Lastly, empirical observation has shown some low condition numbers can occur in the secondary phase when the number of dependent RHS vectors is large. We find that splitting up the total number of RHS vectors into a few smaller blocks alleviates this problem without significantly increasing computational cost. For the problem at hand, we chose to divide the function prediction problems into subproblems with a maximum of 500 RHS vectors. Pseudocode for our algorithm is presented in the Supplementary Materials. 

\subsection{Parameter Tuning}
The multiple RHS framework described in 3.4 lends itself well to parameter tuning, as the different combinations of the parameters $\lambda$ and $\gamma$ described in 3.1 simply yield more RHS label bias vectors to solve for with the same coefficient matrix. 
\\
\indent The original formulation of the GRF objective function in Zhou \textit{et al.} (2004) included the parameter $\mu$, which describes the relative weight to be placed on each component of the objective function, which we formulate as:
\begin{equation}
\min_f \mu\sum (f_i- y_i)^2 + (1-\mu)\sum\sum w_{ij}(f_i - f_j)^2)
\end{equation}	
This parameter was ignored by all previous versions of GeneMANIA, but we test its impact on function prediction by adding it to our tuning methodology.
\\ 
	\indent In order to choose performance-maximizing parameters, we create a synthetic learning problem from the training data, which is characteristically similar to the original learning problem, and choose parameters that yield the best performance on this subproblem. For details on the creation of the synthetic learning problem, and the choice of parameters, please see the Supplementary Details.

\section{Methods}

\subsection{Evaluation Datasets}
We evaluate our algorithm on the MouseFunc benchmark, as well as yeast data, with additional focus on a gold standard data set in yeast. With regard to MouseFunc data, we focus on the Molecular Function branch of the GO hierarchy. For fair comparison to prior work, we use only data available to participants at the time of the MouseFunc exercise: these data include 10 networks (Interpro data, PFAM data, 3 Gene Expression networks, PPI data, Phenotype,  2 Conservation Profile networks, Disease Association data), 1874 molecular function categories, and 21603 mouse genes, with all data gathered in 2006 (see Pena-Castillo \textit{et al.}, 2008). Predictions are made, as in MouseFunc, only for functional categories with between 3 and 300 annotations in the genome, but all functional categories are used in the bias calculation and negative example choice described in sections 3.1 and 3.2.
\\
	\indent For our performance evaluation in yeast we focus on the Biological Process branch of the GO tree, using data obtained from Mostafavi and Morris  (2010), which includes 44 networks of data obtained from BIOGRID (Stark \textit{et al.}, 2006), covering 3904 genes with 1188 biological process categories (which are all categories with between 3 and 300 annotations). We augment this yeast data with experimentally confirmed gold standard annotations in the BP category of GO:0007005, mitochondrion organization and biogenesis (MOB), obtained from Huttenhower \textit{et al.} (2009) (see section 4.3).

\subsection{Functional Association Data}
Association networks are created from feature-based data types using the Pearson Correlation Coefficient, after a frequency transform as described in Mostafavi  \textit{et al.} (2008). Only the top 100 interactions are used for each gene in the training set in order to keep the networks sparse, and a normalization scheme of $W_h^{'} \negmedspace = \negmedspace D_h^{1/2} W_h D_h^{1/2}$ is applied to each network and to the final combined network, where $D_h$ is again the diagonal matrix containing the row sums of $W_h$.

\subsection{Evaluation Frameworks}
For the purposes of evaluation, protein functions are described by GO ontology annotations, observing the common convention of excluding ‘IEA’ annotations.
\\
	\indent As in the MouseFunc competition, performance is evaluated in two different scenarios: 1) a test set where  all GO-annotations are removed from a subset of data (1718 genes in mouse) and then predictions are made from the remaining training data, and 2) a novel set where predictions are made for proteins that have received new annotations at a later date in time. The member genes of this second set consist of the intersection of all proteins that have received at least one new annotation in any of the GO categories for which we are attempting predictions (1954 genes in mouse, 362 genes in yeast), and so include many proteins that already had some annotations in the training set, as well as proteins with no annotations in the training set. 
\\
	\indent We treat the novel scenario as the more important evaluation scheme for this work, because we believe it better reflects the true task facing computational biologists, and is less prone to evaluation biases (Greene and Troyanskaya, 2012). The test set approach suffers from biases stemming from the underlying use of sequence-similarity methods in both input data and GO labeling (discussed in greater detail in the Supplementary Materials), which likely explains the better performance of all algorithms in the test scenario vs. the novel scenario. Error results are presented for the test scenario as well, to facilitate comparison with MouseFunc algorithms. For both the novel and test MouseFunc evaluations, predictions are made for the same set of GO Molecular Function categories as the original competition: 488 and 442 categories respectively. For yeast, we show results only in the novel scenario, with data from June 2007, one year after the training data, which includes 511 GO BP categories with at least one new annotation. 
\\
\indent Please note that any comparison of computational methods using GO annotations as the ground truth suffers from the lack of delineation between negative and absent annotation. This drawback is discussed at length in Huttenhower \textit{et al.} (2009), and can create significant difficulty in evaluating computational prediction methods, as observed false positive predictions may simply be a function of a lack of study rather than incorrect prediction. It is for this reason that performance evaluation in the novel scenario ignores any false positives for genes outside the novel set, as it likely that these genes were not studied at all in the time interval between the annotation date for training and for testing. In order to further alleviate some of the uncertainty caused by incomplete annotation, we present performance evaluation metrics on a ``gold standard'' benchmark of yeast genes experimentally verified by Huttenhower \textit{et al.} 2009 for GO:0007005 mitochondrial organization and biogenesis (MOB). These gold standard annotations include 148 additional positive annotations that match genes in our yeast gene set, and are also added to the novel set used for the general yeast benchmark. Lastly, when calculating performance statistics on the MOB gold standard, we add an additional 2473 genes to the 342 comprising the yeast novel set. These additional genes are the negative examples from Huttenhower \textit{et al.} (2009) that are present in our gene set.

\subsection{Evalutation Metrics: PR vs. ROC, TopScore}
The performance of discriminant-based classification algorithms is most often represented by two plots: The Receiver-Operator Characteristic (ROC) curve, and the Precision-Recall (PR) curve, each of which can be summarized by their AUC, the area that the curve encompasses. While both performance measures attempt to describe how well the ordering of discriminant values captures the true positive and negative labels, each has different strengths and weaknesses. Precision tends to be more easily interpretable for an experimentalist, but averaging $AUC_{PR}$ numbers over many classifiers can be misleading due to the nonlinear nature of precision scores (See Figure S1 in the Supplementary Materials). Conversely $AUC_{ROC}$ provides a better global view of the rankings, but lacks a meaningful interpretation for experimentalists and its magnitude depends on the skew of the dataset. See the Supplementary Materials for a more detailed description of the pitfalls of each metric.
\\
\indent Freitas \textit{et al.} (2010) points out the importance of confidence if computational predictions are to be useful. An experimentalist would like the results of an analysis to help determine the number of predictions to attempt to assay as well as which functions to focus on. In order to create a metric more robust to averaging than PR, but which still enjoys easy interpretability for experimentalists, we propose $\text{TopScore}_c$, defined as:$\frac{\text{(\# true positives $<$ rank c)}}{min(\text{c, positive label count})}$. This score represents the fraction of a fixed number of experiments expected to yield a positive result, normalized by the maximum number of positive results possible. In this paper we present results for $\text{TopScore}_{10}$,  $\text{TopScore}_{100}$, and $\text{TopScore}_{1000}$ for mouse,  and $\text{TopScore}_{10}$,  $\text{TopScore}_{50}$, and $\text{TopScore}_{200}$ for yeast,  providing insight into the usefulness of computational predictions at three different scales of experimental testing (see Supplementary Materials for more details on TopScore). 

\end{methods}

\section{Results and Discussion}
We present results for our proposed techniques based on the evaluation metrics and datasets described in section 4. While no algorithm outperforms every other on all metrics in all evaluation scenarios, our tuned ALBias algorithm shows clear advantages over the current GeneMANIA methods in the vast majority of evaluations. These differences are especially striking in the novel evaluation scenarios, where prior biases play a more important role in the algorithm than in the test scenario. In the yeast proteome, our algorithm achieved a performance increase of 11-26 percentage points in every metric in the novel scenario, while in the mouse proteome we improved all evaluation scores by 2-6 percentage points in the novel scenario.
\\
	\indent It is difficult to dissect the exact cause of the performance increase, as many of our algorithmic changes are interlinked. For example, our choice of negative examples affects both the bias value of selected genes, but also the network combination algorithm, while our bias calculation can also produce genes with a prior of $-1$, but which were not chosen as negative examples for the purposes of network combination. Nevertheless, we have performed additional experiments to shed light on the performance increases, which we present before our full prediction evaluation.

\subsection{Negative Example Choice}
In order to investigate the impact of our novel negative example choice, we evaluate the SW network combination algorithm with no label bias method, using three different negative example methods: the sibling negative examples, setting all non positive genes with GO annotations to negative examples, and our new negative example approach. As shown in Table 1, our negative example choice outperforms previous choices in all three full-organism evaluations, sometimes even approaching the performance of our full SWSN with ALBias and tuned parameters algorithm, indicating that our choice of negative examples is responsible for a significant part of our algorithm's final performance. 
\\
\indent Our negative example choice does not perform better than the AllNeg selection on the yeast gold standard, however, despite the fact that our algorithm decreases the number of validation true positives that are misclassified as negatives from 110 to 6. We attribute the counterintuitive decrease in performance when using ALBNeg in this setting, to the fact that the particular category MOB is specific enough to have small prevalence in the genome (only 5.3\% of yeast genes possess this function), yet it is common enough that many genes have shared an annotation with it, resulting in our algorithm only selecting 691 negative examples. Thus ALLNeg yields high precision by virtue of having so many more negative examples, whereby it avoids false positives, while the rarity of the category means that there are not many true positives, and thus the mislabeling of true positives is outweighed by the decrease in predicted false positives.
\\
	\indent Despite the success of our method in increasing performance in most evaluations, and reducing the instances of mislabeling validation true positives as negatives, in some GO categories this mislabeling still occurs. Accordingly, we believe there is more potential for refined methods that correctly define high-confidence negative examples, and that these methods will have significant impact in the performance of machine learning algorithms. Indeed, we hypothesize that part of the performance gain demonstrated by the earlier HLBias algorithm, was due to the fact that the authors adjusted the labels for all non-positive genes, effectively turning any gene without a label in an ancestral category of the function in question into a negative example. 

\begin{table}[!htb]\centering
\ra{1.3}
\processtable{Performance metrics for different negative example choices: sibling negatives (SibNeg) as in Mostafavi \textit{et al.} (2008), using all non-positive genes with GO annotations as negative (AllNeg), and negative examples based on our ALBias method (ALBNeg). All algorithms were run using the SW network combination method, and the GRF label propagation algorithm of Mostafavi \textit{et al.} (2008). \label{Tab:02}}
{\begin{tabular}{@{}lcccccc@{}}\toprule
\bf{Algorithm} & $\text{\bf{AUC}}_\text{\bf{ROC}}$ & $\text{\bf{AUC}}_\text{\bf{PR}}$ & $\text{\bf{TS}}_\text{\bf{10}}$ & $\text{\bf{TS}}_\text{\bf{100}}^*$ & $\text{\bf{TS}}_\text{\bf{1000}}^*$\\\midrule
\bf{Mouse Novel} \\
\hspace{2mm}  SibNeg & 0.7347 & 0.3236 & 0.4103 & 0.5342 & 0.7411\\
\hspace{2mm} AllNeg &  0.8155 & 0.3420 & 0.4318 & 0.5783 & 0.8354\\
\hspace{2mm} ALBNeg & 0.8366 & 0.3447 & 0.4314 & 0.5793 & 0.8705\\
\bf{Mouse Test}\\
\hspace{2mm} SibNeg & 0.8573 & 0.5019 & 0.6136 & 0.7622 & 0.8725\\
\hspace{2mm} AllNeg & 0.9232 & 0.5168 & 0.6207 & 0.7994 & 0.9530\\
\hspace{2mm} ALBNeg & 0.9330 & 0.5171 & 0.6160 & 0.8014 & 0.9745\\
\bf{Yeast Novel}\\
\hspace{2mm} SibNeg & 0.7566 & 0.3090 & 0.3674 & 0.6014 & 0.8094\\
\hspace{2mm} AllNeg & 0.7563 & 0.2865 & 0.3299 & 0.5284 & 0.8405\\
\hspace{2mm} ALBNeg & 0.8711 & 0.3387 & 0.4133 & 0.7127 & 0.9633\\
\bf{Yeast Gold Standard}\\
\hspace{2mm} SibNeg & 0.7936 & 0.3729 & 0.8 & 0.54 & 0.5068\\
\hspace{2mm} AllNeg & 0.8679 & 0.4685 & 1 & 0.74 & 0.4932\\
\hspace{2mm} ALBNeg & 0.8413 & 0.3896 & 0.7 & 0.6 & 0.4865\\
\bottomrule
\end{tabular}}{*For the yeast scenarios, $\text{TopScore}_{100}$ and $\text{TopScore}_{1000}$ are replaced by $\text{TopScore}_{50}$ and $\text{TopScore}_{200}$}
\end{table}


\subsection{Network Combination Algorithm SWSN}
To examine the effect of our network combination aglorithm, SWSN, we performed a comparison with the SW network weight algorithm, using no label biases and our negative examples, yielding mixed results across evaluation scenarios and metrics. SWSN slightly outperforms SW on the mouse novel set, the two algorithms are virtually tied on the mouse test set, and SW outperforms SWSN on the yeast novel and gold standard evaluations. Yet we believe that further refinement of negative example choice will show SWSN to be a more successful method. In order to demonstrate this, we add to the comparison of the two algorithms in Table 2, a third algorithm (SWSNOracle) in which our negative examples are granted access to a negative oracle, namely the validation annotations, to ensure we do not select any negative examples that are demonstrated positives (there are almost certainly others amongst our negative examples that are true positives but not yet studied at the time of the collection of validation data). This results in stronger performance on the mouse novel set and yeast gold standard, but makes no difference on the mouse test or yeast novel sets, as there were no instances of negative examples that were demonstrated positives in the mouse test and only 11 in the yeast novel benchmark.
\\
\indent We believe this result indicates the promise of our SWSN algorithm, once even better negative example methods are uncovered. Therefore we submit SWSN as having greater intuitive foundation than SW, and the likelihood of future performance gain, despite the fact that it was likely not a significant factor in the current performance increase of our algorithm.

\begin{table}[!htb]\centering
\ra{1.3}
\processtable{Performance metrics for network combination algorithms: Simultaneous weights (SW) from Mostafavi and Morris (2010), our own SWSN algorithm, and SWSN with a negtive oracle (SWSNOracle). All algorithms were run using the GRF label propagation method of Mostafavi \textit{et al.} (2008). \label{Tab:02}}
{\begin{tabular}{@{}lcccccc@{}}\toprule
\bf{Algorithm} & $\text{\bf{AUC}}_\text{\bf{ROC}}$ & $\text{\bf{AUC}}_\text{\bf{PR}}$ & $\text{\bf{TS}}_\text{\bf{10}}$ & $\text{\bf{TS}}_\text{\bf{100}}^*$ & $\text{\bf{TS}}_\text{\bf{1000}}^*$\\\midrule
\bf{Mouse Novel} \\
\hspace{2mm} SW & 0.8366 & 0.3447 & 0.4315 & 0.5793 & 0.8705\\
\hspace{2mm} SWSN & 0.8376 & 0.3460 & 0.4396 & 0.5878 & 0.8755\\
\hspace{2mm} SWSNOracle & 0.8775 & 0.3491 & 0.4433 & 0.6027 & 0.9366\\
\bf{Mouse Test}\\
\hspace{2mm} SW & 0.9330 & 0.5171 & 0.6160 & 0.8014 & 0.9745\\
\hspace{2mm} SWSN & 0.9315 & 0.5177 & 0.6211 & 0.8041 & 0.9684\\
\hspace{2mm} SWSNOracle & 0.9315 & 0.5177 & 0.6211 & 0.8041 & 0.9684\\
\bf{Yeast Novel}\\
\hspace{2mm} SW & 0.8711 & 0.3387 & 0.4133 & 0.7127 & 0.9633\\
\hspace{2mm} SWSN & 0.8632 & 0.3139 & 0.3796 & 0.6294 & 0.9649\\
\hspace{2mm} SWSNOracle & 0.8636 & 0.3139 & 0.3796 & 0.6294 & 0.9656\\
\bf{Yeast Gold Standard}\\
\hspace{2mm} SW & 0.8413 & 0.3896 & 0.7 & 0.6 & 0.4865\\
\hspace{2mm} SWSN & 0.8315 & 0.3729 & 0.7 & 0.52 & 0.5135\\
\hspace{2mm} SWSNOracle & 0.8569 & 0.3871 & 0.7 & 0.54 & 0.5270\\
\bottomrule
\end{tabular}}{*For the yeast scenarios, $\text{TopScore}_{100}$ and $\text{TopScore}_{1000}$ are replaced by $\text{TopScore}_{50}$ and $\text{TopScore}_{200}$}
\end{table}

\subsection{Parameter Tuning Results}
	From the performance measurements presented in section 5.1, we see that while the tuned parameters performed significantly better than the null parameters in the mouse novel and yeast MOB benchmarks, their performance was on par with, or occasionally worse than the null guess in the mouse test and yeast novel benchmarks. We attribute the decrease in performance, primarily in the yeast novel set, to the inherent difference between the state of annotation in yeast and mouse. Our parameter-tuning algorithm was designed to re-create a learning problem where annotations are only partially known, yet in yeast, a well-studied organism, this type of learning problem was most likely not as representative of the true learning problem as it was in mouse, a less-studied organism.
\\
	\indent In general, adapting the tuning process to be representative of the original learning problem is a more intricate problem than first anticipated, and requires further exploration. For detailed analysis of the parameter tuning results, and future avenues of approach, please refer to the Supplementary Materials.


\subsection{Prediction Evaluations}
We present the performance, evaluated by $AUC_{ROC}$, $AUC_{PR}$, and TopScore (TS) metrics, of five algorithms: the original MouseFunc GeneMANIA algorithm, the SW GeneMANIA algorithm presented in Mostafavi and Morris (2010) using sibling negative examples, the SW algorithm combined with the HLBias algorithm of Mostafavi and Morris (2009), and two versions of our algorithm, SWSN with ALBias and naive parameters ($\lambda$=0, $\gamma$=0, $\alpha$=0.5), and SWSN with ALBias and tuned parameters. Results are averaged over all categories, with a closer examination of results broken down by the specificity of the function available in Figure S2.a in the Supplementary Materials.
\\
	\indent In the novel scenario for MouseFunc, our algorithms show a strong increase in performance across all metrics, especially our version with tuned parameters, as seen in Figure 2.a. We see here a large difference in performance between ALBias with tuned parameters and ALBias with naive parameters, indicating that some of our algorithmic performance increase in mouse is due to the ability of our parametric pseudocounting procedure to prevent undue bias influence from understudied GO categories in mouse.

\newlength{\mywidth}
\setlength{\mywidth}{.8\textwidth}
\begin{figure*}[!htb]
\centering
\includegraphics[max size={.95\textwidth}{.95\textheight}]{finalResultsFigure.eps}
\centering
\caption{Performance metrics in (a) the novel scenario in mouse (488 functions, 1954 genes), (b) the test scenario in mouse (442 functions, 1718 genes), and (c) the novel scenario in yeast (511 functions, 342 genes). Metrics are averaged over all GO functions (each with between 3 and 300 counts per genome), and error bars are one standard deviation of the error in the mean.}\label{fig:01}
\end{figure*}

\indent For the mouse test set, the difference in performance is much smaller, because the test set is stripped of all labels, thus negating a key advantage of ALBias (see Figure 2.b for mouse test results). Yet we still see a performance increase in our algorithms across most metrics, due to better biases for genes with edges connected to test genes.
\\
\indent In the yeast novel set we compare all algorithms except the original MouseFunc GeneMANIA algorithm, and observe a striking performance advantage of our algorithms across all evaluation metrics (See Figure 2.c). Further analysis indicates that much of this performance gain is due to our algorithm's incorporation of information from all branches of GO into the label bias calculation. Examining an example GO term "M phase", where our algorithm increased $\text{AUC}_\text{ROC}$ from $0.769$ to $0.892$ and $\text{AUC}_\text{PR}$ from $0.408$ to $0.603$, we find the primary cause to be the validation true positive gene YHR079C-B, which moved from a ranking of 75th to 1st, despite having no Biological Process annotations in the data, thanks to its Cellular Component annotation of "condensed nuclear chromosome", which has a high joint probability with "M phase". Additional examples are provided in the Supplementary Materials.
\\
\indent Lastly on the yeast MOB gold standard (results in Table 3), we see strong performance from our tuned SWSN ALBias algorithm, which achieved significantly higher $AUC_{ROC}$ and  $\text{TopScore}_{200}$ scores, but also from the SW, HLBias algorithm, which achieved the highest $\text{TopScore}_{10}$ and $\text{TopScore}_{50}$ scores, as well as a marginally higher $AUC_{PR}$ score. These results indicate that while our algorithm provided a better global ranking of true positives, the current GeneMANIA algorithm ranked the top true predictions more highly.

\setlength{\textfloatsep}{.25cm}

\begin{table}[!tb]
\processtable{Performance metrics on the yeast gold standard (GO:0007005) with experimental data from Huttenhower \textit{et al.} (2009), comprised of 148 positive examples in 2815 genes. \label{Tab:02}}
{\begin{tabular}{llllll}\toprule
Algorithm & $\text{AUC}_\text{ROC}$ & $\text{AUC}_\text{PR}$ & $\text{TS}_\text{10}$ & $\text{TS}_\text{50}$ & $\text{TS}_\text{200}$\\\midrule
SW, HLBias & 0.8679 & 0.4685 & 1.0 & 0.74 & 0.4932\\
SW, No Bias & 0.7908 & 0.3729 & 0.8 & 0.54 & 0.5068\\
SWSN, Naive Params & 0.8842 & 0.4076 & 0.8 & 0.56 & 0.5068\\
SWSN, Tuned Params & 0.9032 & 0.4634 & 0.7 & 0.70 & 0.5608\\\botrule
\end{tabular}}{}
\end{table}

\subsection{Computational Cost}
	\indent A theoretical complexity analysis of the SBCG algorithm is not possible (see Supplementary Materials), but empirical testing shows a 30\% reduction in the number of flops required for the prediction task on original MouseFunc data, with SBCG converging to a solution with smaller residuals as well. On the yeast benchmark, the reduction in flops was less, at 22\%, due to the fact that the ratio between the number of genes and the number of functions to predict is much smaller (see Table 3 in the Supplementary Materials). As expected, there is an observable increase in computation saved as the number of categories increases, but this is bounded by the fact that our algorithm splits the categories into subsets with a maximum size of 500. This suggests that further computation could be saved by devising a suitable strategy to deal with low condition numbers for larger sets of right-hand-side vectors. Suarjana and Law (1994) suggests that a pre-conditioner applied to the data might help reduce the number of iterations required as well.

\section{Conclusion}
We have addressed several of the key problems facing protein function prediction efforts by proposing novel algorithms, including a method of choosing negative examples, and a parameterized Bayesian methodology for computing prior functional biases from existing annotation data. These methods, applied within the context of the GeneMANIA function prediction algorithm, have resulted in a significant performance increases across several benchmarks. We also introduced a new optimization methodology, which significantly decreased computational costs.
\\
	\indent We devised a framework for tuning parameters in a synthetic novel set which added further performance gain in the novel scenario in mouse, but requires additional work to be more broadly applicable to other evaluation scenarios. Our new SWSN network combination algorithm shows promise if even-better performing negative example methods are developed, though it does not significantly improve performance as of now. Finally, we presented a new evaluation metric that will help experimentalists choose predictions to experimentally verify while still being robust to averaging over multiple function categories.
\\
	\indent When comparing performance statistics of different algorithms, it is important to keep in mind that a difference of a few percentage points can mean hundreds of new true annotations when applied across all functions. For example, a 1\% increase in $\text{TopScore}_{10}$ would result in 187 new true annotations were an experimentalist to use that metric to guide experiments over the 1,874 GO MF categories in the mouse genome (at the time of MouseFunc publication). Thus we believe the algorithms presented here have the potential to guide experimentalists to a large number of fruitful assays, and are in general aligned with current biological understanding of how genes are functionally related to each other through different data types.
\\
	\indent In summary we have shown that our algorithm can perform function prediction through data integration and guilt-by-association with substantially more accuracy and efficiency than previously published algorithms, and provided insight into some of the inherent difficulties still facing the development and evaluation of protein function prediction algorithms.

\section*{Acknowledgement}
We would like to thank Sara Mostafavi for making her code and data for the GeneMANIA algorithm publicly available.

\paragraph{Funding\textcolon} This work was supported by U.S. National Science Foundation grants 0922738, 
0929338, 1158273, and IOS-1126971, and National Institutes of Health GM 32877-21/22, RC1-AI087266, RC4-AI092765, PN2-EY016586, IU54CA143907-01 and EY016586-06.

%\bibliographystyle{natbib}
%\bibliographystyle{achemnat}
%\bibliographystyle{plainnat}
%\bibliographystyle{abbrv}
%\bibliographystyle{bioinformatics}
%
%\bibliographystyle{plain}
%
%\bibliography{Document}


\begin{thebibliography}{}
\bibitem[Drew {\it et~al}., 2011]{Drew11} Drew, K., Winters, P., {\it et~al} (2011) The Proteome Folding Project: proteome-scale prediction of structure and function. {\it Genome Res.}, {\bf 21}, 1981-1994.

\bibitem[Freitas \textit{et~al}., 2010]{Freitas10}
Freitas, A., Wieser, D. C., \textit{et~al}. (2010) On the Importance of Comprehensible Classification Models for Protein Function Prediction. \textit{IEEE/ACM Transactions on Computational Biology and Bioinformatics. }, \textbf{7}(1), 172-182.

\bibitem[Gomez \textit{et~al}., 2003]{Gomez03}
Gomez, S. M., Noble, W. S., \textit{et~al}. (2003) Learning to predict protein- protein interactions. \textit{Bioinformatics}, \textbf{19}, 1875-1881.

\bibitem[Greene and Troyanskaya, 2012]{Greene12}
Greene, CS, and Troyanskaya, OG. (2012) Accurate evaluation and analysis of functional genomics data and methods. \textit{Annals of the New York Academy of Sciences} \textbf{1260(1)}, 95-100. 

\bibitem[Guan \textit{et~al}., 2008]{Guan08}
Guan, Y., Myers, C. L., \textit{et~al}. (2008) Predicting gene function in a hierarchical context with an ensemble of classifiers. \textit{Genome Biol.}, \textbf{9}(Suppl. 1), S3.

\bibitem[Huttenhower \textit{et~al}., 2009]{Huttenhower09}
Huttenhower, C., Hibbs, M. A., \textit{et~al}. (2009) The impact of incomplete knowledge on evaluation: an experimental benchmark for protein function prediction. \textit{Bioinformatics}, \textbf{25}(18), 2404-2410.

\bibitem[Kim \textit{et~al}., 2008]{Kim08}
Kim, W. K., Krumpelman, C., \textit{et~al}. (2008) Inferring mouse gene functions from genomic-scale data using a combined functional network/classification strategy. \textit{Genome Biol.}, \textbf{9}(Suppl. 1), S5.

\bibitem[King \textit{et~al}., 2003]{King03}
King, O. D., Foulger, R. E., \textit{et~al}. (2003) Predicting Gene Function From Patterns of Annotation. \textit{Genome Res.}, \textbf{9}(Suppl. 1), S5.

\bibitem[Lee \textit{et~al}., 2006]{Lee06}
Lee, H., Tu, Z., \textit{et~al}. (2006) Diffusion Kernel-Based Logistic Regression Models for Protein Function Prediction. \textit{OMICS}, \textbf{13}, 896-904.

\bibitem[Leone, 2005]{Leone05}
Leone, M., and Pagnani, A. (2005) Predicting protein functions with message passing algorithms. \textit{Bioinformatics}, {\bf 21}(2), 239-247. doi:10.1093/bioinformatics/bth491.

\bibitem[Marcotte \textit{et~al}., 1999]{Marcotte99}
Marcotte, E. M., Pellegrini, M., \textit{et~al}. (1999) A combined algorithm for genome-wide prediction of protein function. \textit{Nature}, \textbf{402}, 83-86.

\bibitem[Mostafavi \textit{et~al}., 2008]{Mostafavi08}
Mostafavi, S., Ray, D., \textit{et~al}. (2008) GeneMANIA: A real-time multiple association network integration algorithm for predicting gene function. \textit{Genome Biol.}, \textbf{9}(Suppl. 1), S4.

\bibitem[Mostafavi \textit{et~al}., 2009]{Mostafavi09}
Mostafavi, S., and Morris, Q. (2009) Using the Gene Ontology hierarchy when predicting gene function. \textit{UAI Conference Proceedings}, \textbf{2009}.

\bibitem[Mostafavi \textit{et~al}., 2010]{Mostafavi10}
Mostafavi, S., and Morris, Q. (2010) Fast integration of heterogeneous data sources for predicting gene function with limited annotation. \textit{Bioinformatics}, \textbf{26}, 1759-1765.

\bibitem[Obozinski \textit{et~al}., 2008]{Obozinski08}
Obozinski, G., Lanckriet, G., \textit{et~al}. (2008) Consistent probabilistic outputs for protein function prediction. {\it Genome Biol.}, {\bf 9}(Suppl 1.), S6.

\bibitem[Pavlidis, P., and Gillis, J., 2012]{Pavlidis12}
Pavlidis, P., and Gillis, J., (2012) Progress and challenges in the computational prediction of gene function using networks. \textit{F1000 Research} {\bf1(14)}. 

\bibitem[Pe\~na-Castillo {\it et~al}., 2008]{Pena08} Pe\~na-Castillo, L., Tasam, M.,  {\it et~al}  (2008) A critical assessment of Mus musculus gene function prediction using integrated genomic evidence. {\it Genome Biol.}, {\bf 9}(Suppl 1.), S2.

\bibitem[Smoot \textit{et~al}., 2011]{Smoot11}
Smoot, M., Ono, K., \textit{et~al}. (2011) Cytoscape 2.8: new features for data integration and network visualization. \textit{Bioinformatics}, \textbf{27}(3), 431-432.

\bibitem[Stark {\it et~al}., 2006]{Stark06} 
Stark, C., Breitkreutz, B. J. (2006) BioGRID: a general repository for interaction datasets. {\it Nucl. Acids Res.}, {\bf 34}(Suppl. 1), D535-D539.

\bibitem[Suarjana {\it et~al}., 1994]{Suarjana94} 
Suarjana, M., and Law, K. H. (1994) Successive conjugate gradient methods for structural analysis with multiple load cases. {\it Int. Journal for Num. Methods in Eng.}, {\bf 37}(24), 4185-4203.

\bibitem[Tasan \textit{et~al}., 2008]{Tasan08}
Tasan, M., Tian, W., \textit{et~al}. (2008) An en masse phenotype and function prediction system for Mus musculus. \textit{Genome Biol.}, \textbf{9}(Suppl. 1), S8.

\bibitem[Troyanskaya \textit{et~al}., 2003]{Troyanskaya03}
Troyanskaya, O.G., Dolinski, K., \textit{et~al}. (2003) A Bayesian framework for combining heterogeneous data sources for gene function prediction (in Saccharomyces cerevisiae). \textit{Proc. Natl. Acad. Sci.}, \textbf{100}, 8348-8353.

\bibitem[Tsuda \textit{et~al}., 2005]{Tsuda05}
Tsuda, K., Shin, H., \textit{et~al}. (2005) Fast protein classification with multiple networks. \textit{Bioinformatics}, \textbf{21}(Suppl. 2), ii59-ii65.

\bibitem[Qi \textit{et~al}., 2005]{Qi05}
Qi, Y., Klien-Seetharaman, J., \textit{et~al}. (2008) Random forest similarity for protein-protein interaction prediction from multiple sources. \textit{Pac. Symp. Biocomputing}, 531-542.

\bibitem[Zhang \textit{et~al}., 2008]{Zhang08}
Zhang, C., Joshi, T., \textit{et~al}. (2008) An integrated probabilistic approach for gene function prediction using multiple sources of high-throughput data. \textit{Int. Journal of Comp. Biology and Drug Design}, \textbf{1}(3), 254-274.

\bibitem[Zhou \textit{et~al}., 2003]{Zhou04}
Zhou, D., Bousquet, O., \textit{et~al}. (2004) Learning with local and global consistency. \textit{Adv. Neural Inf. Process. Syst.}, \textbf{16}, 321-328.

\bibitem[Zhu \textit{et~al}., 2003]{Zhu03}
Zhu, X., Ghahramani, Z., \textit{et~al}. (2003) Semi-supervised learning using Gaussian fields and harmonic functions. In \textit{ Proceedings of the Twentieth International Conference on Machine
Learning}, Washington DC, USA.

\end{thebibliography}
\end{document}
