-- Demonstrates a way to use Lua to mange C Memory outside
-- that considered by the GC.  This allows nearly normal Lua 
-- code to manage larger amounts of memory  bypassing the GC.
-- Intended to make it feasible to use the speed of lua while 
-- bypassing the downside normally encountered  with GC cleanup. 
-- You have to manually call .done() for the objects to free up 
-- their memory but that is a pretty low overhead to gain access 
-- more of the memory in a 64 bit machine.

local ffi = require("ffi") 
local size_double = ffi.sizeof("double")
local size_char    = ffi.sizeof("char")
local size_float    = ffi.sizeof("float")
ffi.cdef"void* malloc (size_t size);"
ffi.cdef"void free (void* ptr);"
local chunk_size  = 16384 
   -- want big enough chunks that we don't fragment the C 
   -- memory manager
  
-- define a structure which contains a size and array of 
-- doubles where we dynamically allocate the array using 
-- malloc() Do it this way just in case we want to write C
-- code that needs the size.
local DArr = ffi.metatype(
  --               size,               array
  "struct{uint32_t size; double* a;}",
  -- add some methods to our array
  { __index = {
    done = function(self) 
      if self.size == 0  then
        return false
      else
        ffi.C.free(self.a)
        self.a = nil
        self.size = 0
        return true
      end
    end,
    -- copy data element into our externally managed array from the
    -- supplied src array.   Start copying src[beg_ndx], stop copying
    -- at src[end_ndx],  copy into our array starting at self.a[dest_offset]
    copy_in = function(self,  src, beg_ndx, end_ndx, dest_offset)
      -- Can not use mem_cpy because the source is likely
      -- a native lua array.
      print ("self=", self,  " beg_ndx=", beg_ndx, 
         " end_ndx=", end_ndx, "dest_offset=", dest_offset)
      local mydata = self.a
      local dest_ndx  = dest_offset
      for src_ndx = beg_ndx, end_ndx do
         mydata[dest_ndx] = src[src_ndx]
         dest_ndx = dest_ndx + 1
      end
    end,
    -- copy data elements out of our externally managed array to another
    -- array.  Start copying at self.a[beg_ndx] ,  stop copying at self.a[end_ndx]
    -- place elements in dest starting at dest[dest_offset] and working up.
    copy_out = function(self, dest, beg_ndx, end_ndx, dest_offset)
      -- Can not can use mem_cpy because the dest is likely
      -- a native lua array.
      local mydata = self.a
      local dest_ndx  = dest_offset
      for ndx = beg_ndx, end_ndx do
        dest[dest_ndx] = mydata[ndx]
        dest_ndx = dest_ndx + 1
      end
    end,  
      -- return true if I still have a valid data pointer.
    -- return false if I have already ben destroyed.
    is_valid = function(self)
      print("is_valid() size=", self.size, " self.a=", 
        self.a, " self=", self)
      return self.size ~= 0 and  self.a ~= nil
    end,
    
    fill = function(self, anum, start_ndx, end_ndx)
      if end_ndx == nil then
        end_ndx = self.size
      end
      if start_ndx == nil then
        start_ndx = 0
      end 
      local mydata = self.a
      for ndx = 1, end_ndx do
        mydata[ndx] = anum
      end
    end,  -- func fill
    },

    __gc = function(self) 
         self:done()
    end
  }
)  -- end Darr()

-------------------
 --- Constructor for DArr
 ------------------
function double_array(leng)
   -- allocate the actual dynamic buffer.
   local size_in_bytes = (leng + 1) * size_double
   local adj_bytes = (math.floor(size_in_bytes / chunk_size) + 1) * chunk_size
   local adj_len  = math.floor(adj_bytes / size_double)
   
   local ptr = ffi.C.malloc(adj_bytes)
   if ptr == nil then
     return nil
   end
   return DArr( adj_len,  ptr)
end


function avg_arr(tarr, start_ndx, end_ndx)
  local tsum = 0.0
  local num_ele = (end_ndx - start_ndx) + 1
  for x = start_ndx, end_ndx do
    tsum = tsum + tarr[x]
    --print ("tarr[x]=", tarr[x])
  end
  local avg = tsum / num_ele
  return avg
end

function use_up_memory(targetMeg)
   tout ={}
   while  collectgarbage('count') / 1024 <  targetMeg do
      tx = {}
      tout[#tout+1] = tx
      for i = 1, 100000 do
        tx[i] = i + 1.2
      end
   end
   return tout
end

function basic_test()
  -- uncomment call to waste space to see how lua gC interacts
  -- with the external malloc()
  local  waste_space = use_up_memory(100) 
     -- Change this to 1500 to run lua close to limit on it's internal GC. 
     -- Change to 1000 to use up 1 gig which will cause some of the malloc() to fail.
  local num_ele = 75000
  local tmparr = double_array(num_ele)
  -- Note:  Each 75000 array should occupy roughly  600K of RAM.
  --  plus the overhead for our label, size, counter and containing table.
 
  nla  =  {}
  --put something interesting into our native lua object
  local ptr = tmparr.a
  for x = 1, num_ele do
    nla[x] = x
  end
  nlaavg = avg_arr(nla, 1, num_ele)
  print ("nlaavg = ", nlaavg)

   -- put something interesting into our external object
  local ptr = tmparr.a
  for x = 1, num_ele do
    ptr[x] = x
  end
  darravg = avg_arr(ptr,1, num_ele) 
  print("darravg=", darravg)
 
  assert(nlaavg == darravg, 
    "Expected average of nla and dla to be identical and they are not")
 
  -- Demonstrate copying a portion of the lua array into the external buffer
  -- copies elements 100 to 250 into external array into positions
  -- 320 .. 470.
  tmparr:copy_in(nla, 100, 250,  320)
  print ("ptr[320]=", ptr[320])
  assert(ptr[320] == nla[100],   
     "expected ptr[320] to contain" .. nla[100] 
     .. " after the copy_in but received " .. ptr[320])
 
 
  -- Demonstrate copying a portion back to native lua array
  -- copies element 74,950 to 75,000 into dest nla positions
  -- 1  to 50.
  local sbeg = num_ele - 50
  local send = num_ele
  tmparr:copy_out(nla, sbeg,  num_ele, 1)
  print("nla[1] = ", nla[1])
  assert(nla[1] ==  tmparr.a[sbeg], "expected nla[1] to contain ",
           tmparr.a[sbeg], "  after copy_out but got ",nla[1] )
 
 
  print ("pre destroy is_valid=", tmparr:is_valid())
  assert(tmparr:is_valid() == true, 
    "tmparr.is_valid() should be true before the delete")
 
  local dres = tmparr:done() -- destroy and relase our memory
  assert(dres == true,  "First destroy failed")
  print "sucessful destroy"
  print ("post destroy is_valid=",  tmparr:is_valid())
  assert(tmparr:is_valid() == false, 
    "tmparr.is_valid() should be false after done()")
 
  print "try to do second destroy which should not work"
  --- see what happens when we destroy it a second time
  local qres = tmparr:done()
  assert(qres == false, "Second delete should have failed")
  print "second destroy failed as planned"
 
  -- Lets see how much memory we are using when we start.
  print("using ",  collectgarbage('count') / 1024,
    " meg before GC")
  collectgarbage()
  print("using ",  collectgarbage('count') / 1024, 
    " meg after GC") 
  print("Lua GC will not show the externally allocated buffers")
  -- TODO:  Figure out FFI Call to get total process memory from Windows.
 
 -- for num_pass = 1,50 do
    -- Now try to create a enough of the external arrays that it would normally
    -- crash Lua.
    local taget_num = 3000000000 -- 3.0 gig
    local array_size_bytes  = num_ele * size_double 
    local num_arr_to_create = math.floor(taget_num / array_size_bytes) + 1
    print ("attempting to create ", num_arr_to_create,
            " arrays each ", array_size_bytes, 
            " bytes in size")
    local tholder = {}
    for andx = 1, num_arr_to_create do        
      local da =  double_array(num_ele)
      if da == nil then
        local mba = (andx * array_size_bytes) / 1000000
        local lua_mem = collectgarbage('count') / 1024
        local lua_mem = math.floor(lua_mem * 100) / 100
        local tot_meg = mba + lua_mem
        print ("failed to create andx=", andx,  
                " mb attempt=", mba,  
                " total Meg Used=", tot_meg)
      else
        --print ("create andx=", andx, " da=", da, " da.a=", da.a)
        da:fill(andx)
        tholder[andx] = da  
      end
    end
    print "finished create"
    print("using ",  collectgarbage('count') / 1024, 
       " meg before GC")
    collectgarbage()
    print("using ",  collectgarbage('count') / 1024, 
           " meg after GC") 

  
    print "dbl check access every element dbl avg"
    for andx = 1, num_arr_to_create do
      local da = tholder[andx]
      if da ~= nil then
        local darravg = avg_arr(da.a,1, num_ele)
        --print("andx=", andx, " da=", da, "da.a=", 
        --         da.a, " darravg = " , darravg)
        local calcavg = andx 
           -- we know we filled each array with it's index 
           -- value so that is what the average should be.
        local roundavg = math.floor(darravg * 1000) / 1000 
            -- have to round because of accumulated floating 
            -- point error
        assert(roundavg == calcavg,  
           " avg failed expected " .. calcavg ..  " got " 
           .. roundavg .. " ndx=" .. andx)
      end
    end
    print "Finished access check"
    print ("using ",  collectgarbage('count') / 1024, " meg before GC")
    collectgarbage()
    print ("using ",  collectgarbage('count') / 1024, " meg after GC")

  
    print "Start destroying our external arrays"
    for andx = 1, num_arr_to_create do
      local da = tholder[andx]
      if da ~= nil then
        --print ("destroy andx=", andx, " da=", da, " da.a=", da.a)  
        local deleteok = da:done()
        assert(deleteok == true, 
           " delete failed array #" .. andx)
      end
    end
    tholder = nil
    print "Finished destroy"
    print ("using ",  collectgarbage('count') / 1024, 
      " meg before GC")
    collectgarbage()
    print ("using ",  collectgarbage('count') / 1024, 
       " meg after GC")
    print (" We expect the GC to reclaim some here because"
           .. " we free up the space in our container array")
 -- end
end -- func


------------------------
---- MAIN ----------
------------------------
if arg[0] == "ffi_non_gc_double_array_v2.lua"  then
   basic_test()
end