/ getpredictions takes the weights learned and 
/ applies them to the last year.
/ Gives predictions and actual answers (which will be used just to check).
getpredictions:{[mystock; otherkeys; w];
	i: mykeys ? mystock;
	S: (1 + (floor mynumiter * n)) _ (mytradeflipreturns[i]); 
	Xflipped: ();
	k: 0;
	while[k < count otherkeys;
		i: mykeys ? otherkeys[k];
		Xflipped,: enlist (-1) _ ((floor mynumiter * n)) _ (mytradeflipreturns[i]);
			/ ignore the very last of this
		k+: 1;
	];
	X: flip Xflipped; / X[j] are all predictors for S[j]
	alldots: dotprod[w] each X;
	:(alldots; S)
  }


/ Find the commission of each of mystocks based on dollars allocated
/ to that stock and price of stock on that day.
/ start is the offset in days
/ Side effect is to update the position vectors. 
/ We pay commission only on the incremental change done.
/ Otherwise commission is too high.
findcommission:{[mystocks; i; dollarsalloc]
	mydollarsalloc: dollarsalloc;
	start: ((floor mynumiter * n));
	comm: ();
	indexesused: ();
	allprices: ();
	j: 0;
	while[j < count mystocks;
		k: mykeys ? mystocks[j]; / mykeys and positionstocks are same
		positionnow[k]:: positionnext[k];
  		x: (mydollarsalloc[j]) % (mytradeflipprices[k;i+start] );
		x: myround[x; 100.0];
		mydollarsalloc[j]: x * (mytradeflipprices[k;i+start] );
			/ change dollars allocated based on rounding
		allprices,: mytradeflipprices[k;i+start];
  		positionnext[k]:: x;
		indexesused,: k;
		newshares: abs (positionnext[k] - positionnow[k]);
  		comm,: 0.005 * newshares;
		j+: 1;
	];
 	:(comm; indexesused; allprices; mydollarsalloc)
 }

/ For day i,
/ given a set of predictions about mystocks, allocate bets based on those
/ and then calculate profit.
/ Sort the predictions and buy the predictions above the average
/ and sell the predictions below the average
allocateandbet:{[mystocks; i; mypredictions; myresults; amttobet]
	posprofit: 0.0;
	negprofit: 0.0;
	s: asc mypredictions;
	a: s[floor (count s) % 2]; / find median prediction
	indexesused: ();
	allprices: ();
	commission: 0;
	ii: where mypredictions > a;
	if[0 < count ii;
	  pospredictions: mypredictions[ii] - a;
	  posalloc: pospredictions % (sum pospredictions);
	  dollarsalloc: posalloc * amttobet;
	  myquad: findcommission[mystocks[ii]; i; dollarsalloc];
	  commission+: sum myquad[0];
		/ this is the commission of 5 cents per share
	  indexesused,: myquad[1];
	  allprices,: myquad[2];
	  dollarsalloc: myquad[3];
	  posprofit: dotprod[dollarsalloc; myresults[ii]]; 
	];
	jj: where mypredictions < a;
	if[0 < count jj;
	 negpredictions: a - mypredictions[jj]; / more negative get more weight
	 negalloc: negpredictions % (sum negpredictions);
	 dollarsalloc: negalloc * amttobet;
	 myquad: findcommission[mystocks[jj]; i; dollarsalloc];
	 commission+: sum myquad[0];
		/ this is the commission of 5 cents per share
	 indexesused,: myquad[1];
	 allprices,: myquad[2];
	 dollarsalloc: myquad[3];
	 negprofit: neg dotprod[dollarsalloc; myresults[jj]]; 
	];
	overallprofit: posprofit + negprofit; / ignores commission so far
	allindexes: til count positionnext;
	xx: difference[allindexes; indexesused];
	if[0 < count xx ;
		positionnext[xx]:: 0.0; / no action on these stocks
		totsharestobezeroed: sum abs each positionnow[xx];
		commission+: 0.005 * totsharestobezeroed;
		positionnow[xx]:: 0.0; / zero out these positions 
	]; 
	if[(0 < count ii) & (0 < count jj); :(overallprofit-commission; commission; allprices);];
	5 + `x; / should not get here
	:0.0
 }

/ for each stock, we are given the weights
/ Now we will take each day and allocate both buys and sells based
/ on predicted ups and downs.
findandapplyall:{[stockweights; initialamount]
   n: count stockweights;
   predictionresult: (); / will hold the predicted returns and the real ones
		/ Each entry is a list of predicted returns 
		/ followed by a list of real ones.
   i: 0;
   while[i < n; / number of stocks
     mystock: stockweights[i;0];
     otherkeys: stockdir[mystock];
     w: stockweights[i;1];
     predictionresult,: enlist getpredictions[mystock; otherkeys; w];
     i+: 1;
   ];
   predictions: flip predictionresult[;0]; / a list of list of predictions
	/ In the flip
	/ each list of predictions pertains to one day. 
	/ Each entry in that list is for each stock in the order of stockweights
   results: flip predictionresult[;1]; / list of list of results in same format
   dailyprofit: ();
   i: 0;
   runningamount: ();
   amttobet: initialamount;
   runningamount,: amttobet;
   while[i < count predictions; / the number of days
     x: allocateandbet[stockweights[;0]; i; predictions[i]; results[i]; amttobet];
     amttobet+: x[0];
     runningamount,: amttobet;
     dailyprofit,: enlist x;
     i+: 1;
   ];
   6 + `x; / amttobet
   :dailyprofit[;0];
 }

/ DATA
/ EXECUTION

/ mytradeflip: flip mytrade / don't need initial data any more
mytradeflip: get `:mytradeflipfile 
x: first exec mytradedate from latest 
addin: not x in mytradeflip[`tradedate];
if[1b = addin;
 mytradeflip[`tradedate],: x;
 ];
tradedates: 1 _ mytradeflip[`tradedate]
mykeys: 1 _ key mytradeflip
(` $ ":paolostocks") 0: enlist spitcomma mykeys
positionstocks: mykeys
positionnow: (count positionstocks) # 0.0
positionnext: (count positionstocks) # 0.0
mytradeflipreturns: ()
myprices: ()
i: 0
while[i < count mykeys;
  y: mykeys[i];
  if[addin;
    x: exec myprice from latest where mykey = y;
    mytradeflip[mykeys[i]],: x;
  ];
  mytradeflipreturns,: enlist (1 _ deltas mytradeflip[mykeys[i]]) % ((-1) _ mytradeflip[mykeys[i]]);
  mytradeflipprices,: enlist ((-1) _ mytradeflip[mykeys[i]]);
	/ entry k in each price list is the price at time period k
	/ entry k in each returns list is the return from day k to k+1
	/ list i pertains to mykeys[i]
  i+: 1;
  ];
`:mytradeflipfile set mytradeflip;
n: 250;
mynumiterorig: floor (count mytradeflipreturns[0]) % n;
numyearsoftesting: 1; // ??? can be changed
mynumiter: mynumiterorig - numyearsoftesting; 
// We will use this data to train the model. Then we will run it on the
// last year.

/ first part: find highly correlated pairs
mypairsall: ();
lowestcorr: ();
k: 0;
while[k < mynumiter;
 mytrips: ();
 i: 0;
 while[i < ((count mykeys) - 1);
  j: i+1;
  while[j < count mykeys;
	x1: (k*n) _ mytradeflipreturns[i];
	x2: (k*n) _ mytradeflipreturns[j];
	c: cor[x1[til n]; x2[til n]];
	if[c > 0.70; mytrips,: enlist mykeys[i], mykeys[j], c];
	j: j+1;
   ];
   i: i+1;
  ];
   mypairsall,: findtop[200; mytrips];
   k+: 1;
 ];
 x: findinter[mypairsall];
 stock1: x[;0];
 stock2: x[;1];
 numberyears: x[;2];
 stockpairs:([]stock1;stock2;numberyears);
 save `:stockpairs.csv;
 highstockpairs: select from stockpairs where numberyears >= (5-numyearsoftesting);
 / ?? Above are the pairs that are highly correlated all previous years.
 stockdir: exec stock1 by stock2 from highstockpairs;
 stockdir,: exec stock2 by stock1 from highstockpairs;
 / Those are the ones we are going to make predictions about.
 / Part 2: run machine learning on each stock
 mytradedates: ((floor mynumiter * n)) _ tradedates;
 / The following tries to give weights to different stocks, but uses
 / stochastic gradient descent.
 stockweights: findweights each ((key stockdir));
 dailyprofits: findandapplyall[stockweights; initialamount];
(` $ ":profitstoc") 0: string each dailyprofits;
 show enlist ("Number of days: "), string count dailyprofits;
 show enlist ("Stochastic gradient descent profit without compounding: "), string sum dailyprofits;
 compoundprofit: amttobet * prd (1+(dailyprofits%amttobet));
 show enlist ("Stochastic gradient descent profit with compounding: "), string compoundprofit;
 show enlist ("Sharpe Ratio over whole period compounded: "), string compoundprofit;
 show enlist ("Best day return: "), string max dailyprofits;
 show enlist ("Worst day return: "), string min dailyprofits;

 / The following is similar but uses correlation.
 / p: pairtrade[key stockdir]
 / show enlist ("Pairs trading profit: "), string p