/ Generate data for the contract_item table, inventorybig and inventorysmall$
/ Then generate numtests test sales.
/ These are updates for inventorybig
/ These are selects followed by updates for inventorysmall if present
/ and they are selects followed by inserts when they are not.
/ Replication testing:

/ Generate some string data types.
/ Updates & Inserts.

/ All relevant parameters can be found under PARAM


/ BASIC ROUTINES



/ find difference between list[0] and list[1]
listdiff:{[list]
  :differ[list[0]; list[1]]
}

/ returns one if x is a subset of y
subset:{[x; y]
  i: y ?/: x
  : ~ (#y) _in i
}

/ returns one if x is a subset of y
subset:{[x;y] (#y) > |/ y ?/: x}        

differ:{[x;y]
  x,: ()
  y,: ()
  i: y ?/: x
  j: & i = #y
  :?x[j]
}

/ A faster difference, yielding indexes in x that differ from y
differindexes:{[x;y]
  i: y ?/: x
  j: & i = #y
  :j
}


/finds intersection of two lists
/ fastest of all
intersect: {[x;y]
  x,: ()
  y,: ()
  i: x ?/: y
  :x[(?i) _dv #x]
}

/finds intersection of two lists
/ fastest of all
intersect: {[x;y]
  x,: ()
  y,: ()
  if[(#x) < (#y)
    i: x ?/: y
    j: & i < #x
    :x[?i[j]]
  ]
  i: y ?/: x
  j: & i < #y
  :y[?i[j]]
}

/finds intersection of two lists
/ fastest of all
hasintersect: {[x;y]
  x,: ()
  y,: ()
  i: x ?/: y
  : (&/i) < #x
}

/ x is a proper subset of y
propersubset:{[x;y]
  x,: ()
  y,: ()
  if[~ (#x) < (#y); :0] / must be smaller
  :(#x) = (#intersect[x;y])
}

/ x is a proper subset of y
propersubset:{[x;y]
  x,: ()
  y,: ()
  if[~ (#x) < (#y); :0] / must be smaller
  :subset[x;y]
}

/finds indexes in x and y that intersect
/ If x and y are both sets, then the results will be of the same length
/ fastest of all
intersectindexes: {[x;y]
  i: x ?/: y / where each y hits
  j: & i < #x / those ys that hit
  :(i[j];j)
}

/finds indexes in x that intersect with y
intersectleftindexes: {[x;y]
  i: x ?/: y / where each y hits
  j: & i < #x / those ys that hit
  :i[j]
}

/finds intersection of two lists
/ and returns index pairs of matches. Assumes no duplicates
/ in either list
intersectbothindexes: {[x;y]
  x,: ()
  y,: ()
  i: x ?/: y
  pairs: (i ,' (!#y))
  k: & pairs[;0] < #x
  :pairs[k]
}

/ finds intersection of two lists that may have duplicates
bagintersectbothindexes:{[x;y]
  alreadyused: out: ()
  i: 0
  while[i < #x
	my: x[i]
	jj: & y ~\: my
	jj: differ[jj; alreadyused]
	if[0 < #jj
		out,: ,(i;*jj)
		alreadyused,: *jj
	]
	i+: 1
  ]
  :out
}

/ intersect many lists
multiintersect:{[lists]
 size: #lists
 if[2 > size; :lists]
 first: lists[0],()
 jj: ,/ ?:' first (?/:)/: lists[1+ !(size-1)] / find indexes in first
 x: @[(1+#first) # 0; jj; + ; 1]
 x: (-1) _ x / delete missing entry
 kk: & x = size - 1
 :first[kk]
}

/ this is a set intersection so we remove duplicates
multiintersect:{[lists]
 size: #lists
 if[2 > size; :lists]
 first: lists[0],()
 jj:  first ?/: (,/ ?:' lists[1+ !(size-1)]) / find indexes in first
 x: @[(1+#first) # 0; jj; + ; 1]
 x: (-1) _ x / delete missing entry
 kk: & x = size - 1
 :first[kk]
}

avg:{(+/ x) % # x}

var:{avg[_sqr x] - _sqr avg[x]}
std:{_sqrt var[x]}
cov:{avg[x * y] - avg[x] * avg[y]}
corr:{ (cov[x;y])%((std[x]) * (std[y]))}
/ delay based search
corrdelay:{[delay;x;y] 
  x: (-delay) _ x
  y: delay _ y
  (cov[x;y])%((std[x]) * (std[y]))}


/ END BASICS

/ APPLICATION SPECIFIC

spitvert:{[list] ,/ ($list) ,\: ("|")}
spitcomma:{[list] x: (-1) _ ,/ ($list) ,\: (",")
  :x,(")")}
spitcommaexec:{[list] x: (-1) _ ,/ ($list) ,\: (",")
  :x,(");")}

/ PARAMETERS

numtests: 10000
numcontracts: 10000
numsmallstep: numcontracts * 1
numbigstep: 10 * numsmallstep
numsteps:  50 / number of millions of rows in bigtable
if[0 < #_i
  numsteps: 0 $ _i[0]
]
` 0: ,"k gendata numberofhundredsofthousandsofrows"

/ EXECUTION


contractids: 1 + !numcontracts
avails: numcontracts # ,"T"
accoms: numcontracts _draw 100
committed: ("TF")[numcontracts _draw 2]

/ x: "insert contract_item values("
/ out:(x),/:(spitvert'ids,'avails,'accoms,'committed),\:(,")")

"contract_item_file" 0: spitvert'contractids,'avails,'accoms,'committed

/ What I should do is lay out the big ones on consecutive days
/ and then have the small take samples of those.
/ Then I need a mixture of selects and updates and selects and inserts.

savecontract: ()
savenight: ()
lastid: 1
lastsmallid: 1
lastnight: 1
istep: 0
while[istep < numsteps

  ids: lastid + !numbigstep
  lastid+: numbigstep / for next time
  avails: numbigstep # ,"T"
  contract_item_id: ()
  night: ()
  i: 0
  while[i < #contractids
	contract_item_id,: (_ numbigstep % numcontracts) # contractids[i]
	night,: lastnight+!(_ numbigstep % numcontracts) 
	i+: 1
  ]
  lastnight+: (_ numbigstep % numcontracts)
  quantity: numbigstep # 10
  x: spitvert'ids,'avails,'contract_item_id,'night,'quantity
  filename: ("inventorybig_file" ), ($istep+1)
  filename 0: x

  bignight: night
  bigcontract_item_id: contract_item_id
  ids: lastsmallid + !numsmallstep
  lastsmallid+: numsmallstep
  avails: numsmallstep # ,"T"
  ii: numsmallstep _draw -numbigstep
  contract_item_id: bigcontract_item_id[ii] / taken randomly from above
  night: bignight[ii]
  quantity: quantity[ii]

  smallcontract_item_id: contract_item_id
  smallnight: night

  x: spitvert'ids,'avails,'contract_item_id,'night,'quantity
  filename: ("inventorysmall_file" ), ($istep+1)
  filename 0: x
  if[istep < 10
	savecontract,: bigcontract_item_id
	savenight,: bignight
  ]
  istep+: 1
]



/ The big test is just numtests number of updates.
ii:  numtests _draw - _ #savecontract
testcontract: savecontract[ii]
testnight: savenight[ii]
x: "update inventorybig set quantity = quantity - 1 where contract_item_id = "
out: x ,/: ($testcontract) ,' (" and night = "),/: ($testnight) ,\:("; ")
"bigtest" 0: out

x:"EXECUTE PROCEDURE bigupdate("
x:"SELECT bigtest("
out:(x),/:(spitcommaexec'contract_item_id,'night)
"bigtestproc" 0: out


avails: (#testcontract) # ,`"'T'"
contract_item_id: testcontract / taken randomly from above
night: testnight
quantity: (#testcontract) # 10
x:"EXECUTE PROCEDURE smalltest("
x:"SELECT smalltest("
out:(x),/:(spitcommaexec'avails,'contract_item_id,'night,'quantity)


"smalltest" 0: out

\\