-- Demonstrates a way to use Lua to mange C Memory outside that
-- managed by the GC.  This allows nearly normal Lua code to manage
-- larger amounts of memory  bypassing the GC.   Intended to make
-- it feasible to use the speed of lua while bypassing the downside 
-- normally encountered  with GC cleanup.     You have to manually 
-- call .done() for the objects to free up their memory but that is 
-- a pretty low overhead to gain access more of the memory in a 
-- 64 bit machine.

local ffi = require("ffi") 
local size_double = ffi.sizeof("double")
local size_char    = ffi.sizeof("char")
local size_float    = ffi.sizeof("float")
ffi.cdef"void* malloc (size_t size);"
ffi.cdef"void free (void* ptr);"
local chunk_size  = 16384 -- want big enough chunks that we don't 
  --fragment the C memory manager
  
  -- define a structure which contains a size and array of doubles
  -- where we dynamically allocate the array of doubles.   Do it this
  -- way just in case we want to write C code that needs the size.
   local DArr = ffi.metatype(
        --               size,               array
        "struct{uint32_t size; double* a;}",
        {__gc = function(self) ffi.C.free(self.a) end}
    )  
  
function double_array(length, label)
   local self = {}
   if label ~= nil then
     self.label = label 
     -- self.label is simply for diagnostics 
   else
     self.label = ""
   end
 
   -- allocate the actual dynamic buffer.
   local  adj_len  = ((math.floor(length / chunk_size) + 1) * chunk_size)
   local size_in_bytes = size_double * (adj_len  + 1) 
   ptr = ffi.C.malloc(size_in_bytes)
   if ptr == nil then
     return nil
   end
   assert(ptr ~= nil,  
             "Out of memory trying to alloc " 
             .. size_in_bytes .. " for " .. self.label)
   self.data = DArr( adj_len,  ptr)
   self.a = self.data.a 
   --- NOTE: Would be great if we could add these 
   --  methods directly to  the self.data directly and get 
   -- rid of the extra layer of table.  That way we could 
   -- simply index it directly rather than copy out ptr 
   -- to a local variable.
 
   -- destroy our array and free up the memory.  Returns true
   -- if successful otherwise will return false. when structure
   -- has already been destroyed.
   function self.done()   
     if (self.data == -1) then
       return false
     else
       self.a = nil
       ffi.C.free(self.a)      
       self.data.a = nil
       self.data = -1      
       return true
     end
   end
   
  -- copy data element into our externally managed array
  -- from the supplied src array.   Start copying  at
  -- src[beg_ndx], stop copying at src[end_ndx],  copy into
  -- our array starting at self.a[dest_offset]
  function self.copy_in(src, beg_ndx, end_ndx, dest_offset)
   -- I don't think we can use mem_cpy because the source is likely
   -- a native lua array.
    local mydata = self.a
    local dest_ndx  = dest_offset
    for src_ndx = beg_ndx, end_ndx do
      mydata[dest_ndx] = src[src_ndx]
      dest_ndx = dest_ndx + 1
    end
  end
 
  -- copy data elements out of our externally managed array 
  -- to another array.  Start copying at self.a[beg_ndx] ,  
  -- stop copying at self.a[end_ndx] place elements in 
  -- dest starting at dest[dest_offset] and working up.
  function self.copy_out(dest, beg_ndx, end_ndx, dest_offset)
  -- I don't think we can use mem_cpy because the dest is likely
  -- a native lua array.
    local mydata = self.a
    local dest_ndx  = dest_offset
    for ndx = beg_ndx, end_ndx do
      dest[dest_ndx] = mydata[ndx]
      dest_ndx = dest_ndx + 1
    end
  end 
 
  -- return true if I still have a valid data pointer.
  -- return false if I have already ben destroyed.
  function self.is_valid()
    return self.a ~= nil
  end
 
  function self.fill(anum, start_ndx, end_ndx)
    if end_ndx == nil then
      end_ndx = self.data.size
    end
    if start_ndx == nil then
      start_ndx = 0
    end 
    local mydata = self.a
    for ndx = 1, end_ndx do
      mydata[ndx] = anum
    end
  end -- func fill
  return self
end -- func constructor


function avg_arr(tarr, start_ndx, end_ndx)
  local tsum = 0.0
  local num_ele = (end_ndx - start_ndx) + 1
  for x = start_ndx, end_ndx do
    tsum = tsum + tarr[x]
    --print ("tarr[x]=", tarr[x])
  end
  local avg = tsum / num_ele
  return avg
end

function use_up_memory(targetMeg)
   tout ={}
   while  collectgarbage('count') / 1024 <  targetMeg do
      tx = {}
      tout[#tout+1] = tx
      for i = 1, 100000 do
        tx[i] = i + 1.2
      end
   end
   return tout
end

function basic_test()
  -- uncomment call to waste space to see how lua gC interacts
  -- with the external malloc()
  local  waste_space = use_up_memory(100) 
     -- Change this to 1500 to run lua close to limit on it's internal GC. 
     -- Change to 1000 to use up 1 gig which will cause some of the malloc() to fail.
  local num_ele = 75000
  local tmparr = double_array(num_ele, "my test data arr")
  -- Note:  Each 75000 array should occupy roughly  600K of RAM.
  --  plus the overhead for our label, size, counter and containing table.
 
  nla  =  {}
  --put something interesting into our native lua object
  local ptr = tmparr.a
  for x = 1, num_ele do
    nla[x] = x
  end
  nlaavg = avg_arr(nla, 1, num_ele)
  print ("nlaavg = ", nlaavg)
 
 
   -- put something interesting into our external object
  local ptr = tmparr.a
  for x = 1, num_ele do
    ptr[x] = x
  end
  darravg = avg_arr(ptr,1, num_ele) -- our darr does not respond to the # operator.
  print("darravg=", darravg)
 
  assert(nlaavg == darravg, "Expected average of nla and dla to be identical and they are not")
 
  -- Demonstrate copying a portion of the lua array into the external buffer
  -- copies elements 100 to 250 into external array into positions
  -- 320 .. 470.
  tmparr.copy_in(nla, 100, 250,  320)
  print ("ptr[320]=", ptr[320])
  assert(ptr[320] == nla[100],   "expected ptr[320] to contain" .. nla[100] .. " after the copy_in but received " .. ptr[320])
 
 
  -- Demonstrate copying a portion back to native lua array
  -- copies element 74,950 to 75,000 into dest nla positions
  -- 1  to 50.
  tmparr.copy_out(nla, num_ele - 50,  num_ele, 1)
  print("nla[1] = ", nla[1])
  assert(nla[1] == 74950, "expected nla[1] to contain 74950 after copy_out")
 
 
  print ("pre destroy is_valid=", tmparr.is_valid())
  assert(tmparr.is_valid() == true, "tmparr.is_valid() should be true before the delete")
 
  local dres = tmparr.done() -- destroy and relase our memory
  assert(dres == true,  "First destroy failed")
  print "sucessful destroy"
  print ("post destroy is_valid=",  tmparr.is_valid())
  assert(tmparr.is_valid() == false, "tmparr.is_valid() should be false after delete")
 
  print "try to do second destroy which should not work"
  --- see what happens when we destroy it a second time
  local qres = tmparr.done()
  assert(qres == false, "Second delete should have failed")
  print "second destroy failed as planned"
 
  -- Lets see how much memory we are using when we start.
  print("using ",  collectgarbage('count') / 1024, " meg before GC")
  collectgarbage()
  print("using ",  collectgarbage('count') / 1024, " meg after GC") 
  print("Lua GC will not show the externally allocated buffers")
  -- TODO:  Figure out FFI Call to get total process memory from Windows.
 
 -- for num_pass = 1,50 do
    -- Now try to create a enough of the external arrays that it would normally
    -- crash Lua.
    local taget_num = 3000000000 -- 3.0 gig
    local array_size_bytes  = num_ele * size_double                                         
    local num_arr_to_create = math.floor(taget_num / array_size_bytes) + 1
    print ("attempting to create ", num_arr_to_create, " arrays each ", array_size_bytes, " bytes in size")
    local tholder = {}
    for andx = 1, num_arr_to_create do  
      --local da = double_array(num_ele, "my test data arr " .. andx)
      local da =  double_array(num_ele, "my test data arr" .. andx)
      if da == nil then
        mba = (andx * array_size_bytes) / 1000000
        lua_mem = collectgarbage('count') / 1024
        lua_mem = math.floor(lua_mem * 100) / 100
        tot_meg = mba + lua_mem
        print ("faiiled to create andx=", andx,  " mb attempt=", mba,  " total Meg Used=", tot_meg)
      else
        --print ("create andx=", andx, " da=", da, " da.a=", da.a)
        da.fill(andx)
        tholder[andx] = da  
      end
    end
    print "finished create"
    print("using ",  collectgarbage('count') / 1024, " meg before GC")
    collectgarbage()
    print("using ",  collectgarbage('count') / 1024, " meg after GC") 
  
  

  -- Make sure we could access every item of every created array.
    print "Make sure we can access every element in every array dbl check with avg"
    for andx = 1, num_arr_to_create do
      local da = tholder[andx]
      if da ~= nil then
        local darravg = avg_arr(da.a,1, num_ele)
        --print("andx=", andx, " da=", da, "da.a=", da.a, " darravg = " , darravg)
        local calcavg = andx -- we know we filled each array with it's index value so that is what the average should be.
        local roundavg = math.floor(darravg * 1000) / 1000 -- have to round because of accumulated floating point error
        assert(roundavg == calcavg,  " avg failed expected " .. calcavg ..  " got " .. roundavg .. " ndx=" .. andx)
      end
    end
    print "Finished access check"
    print ("using ",  collectgarbage('count') / 1024, " meg before GC")
    collectgarbage()
    print ("using ",  collectgarbage('count') / 1024, " meg after GC")

  
    print "Start destroying our external arrays"
    for andx = 1, num_arr_to_create do
      local da = tholder[andx]
      if da ~= nil then
        --print ("destroy andx=", andx, " da=", da, " da.a=", da.a)  
        local deleteok = da.done()
        assert(deleteok == true, " delete failed array #" .. andx)
      end
    end
    tholder = nil
    print "Finished destroy"
    print ("using ",  collectgarbage('count') / 1024, " meg before GC")
    collectgarbage()
    print ("using ",  collectgarbage('count') / 1024, " meg after GC")
    print (" We expect the GC to reclaim some here because we free up the space in our container array")
 -- end
end -- func


------------------------
---- MAIN ----------
------------------------
if arg[0] == "ffi_non_gc_double_array.lua"  then
   basic_test()
end