From a26a1500df31858f5962ac3dc722c276c8976d3a Mon Sep 17 00:00:00 2001 From: Nathan DECHER Date: Mon, 14 Sep 2020 09:44:30 +0200 Subject: [PATCH] added fs cache, closes #9 --- bin/moonbuild.moon | 83 ++++++++++++----------- moonbuild.lua | 149 ++++++++++++++++++++++++++++++++++++++--- moonbuild/fscache.moon | 33 +++++++++ moonbuild/fsutil.moon | 44 ++++++++++-- 4 files changed, 252 insertions(+), 57 deletions(-) create mode 100644 moonbuild/fscache.moon diff --git a/bin/moonbuild.moon b/bin/moonbuild.moon index fdf1b3b..8a044f1 100755 --- a/bin/moonbuild.moon +++ b/bin/moonbuild.moon @@ -1,43 +1,44 @@ #!/usr/bin/env moon -argparse=require 'argparse' +argparse = require 'argparse' require 'moonscript' import loadfile from require 'moonscript.base' import truncate_traceback, rewrite_traceback from require 'moonscript.errors' import trim from require 'moonscript.util' -util=require 'moonbuild.util' +util = require 'moonbuild.util' +import freezecache, invalidatecache from require 'moonbuild.fsutil' import exists, mtime, run, min, max, first, flatten, match, patsubst, sortedpairs from util import insert, concat from table -parser=argparse 'moonbuild' +parser = argparse 'moonbuild' parser\argument('targets', "Targets to run")\args '*' parser\flag '-a --noskip', "Always run targets" parser\flag '-l --list', "List available targets" parser\flag '-d --deps', "List targets and their dependancies" -args=parser\parse! +args = parser\parse! -- util functions -loadwithscope= (file, scope) -> - fn, err=loadfile file +loadwithscope = (file, scope) -> + fn, err = loadfile file error err or "failed to load code" unless fn - dumped, err=string.dump fn + dumped, err = string.dump fn error err or "failed to dump function" unless dumped load dumped, file, 'b', scope -pcall= (fn, ...) -> - rewrite=(err) -> - trace=debug.traceback '', 2 - trunc=truncate_traceback trim trace - rewrite_traceback trunc, err +pcall = (fn, ...) -> + rewrite = (err) -> + trace = debug.traceback '', 2 + trunc = truncate_traceback trim trace + (rewrite_traceback trunc, err) or trace xpcall fn, rewrite, ... -- command object -- represents a command that can be called class Command new: (@cmd, ...) => - @args={...} + @args = {...} __unm: => @run error: true, print: true __len: => @run error: true @@ -49,11 +50,11 @@ class Command -- build object -- represents a target class BuildObject - all={} - skip={} + all = {} + skip = {} @find: (name) => - target=all[name] + target = all[name] return target if target for glob, tgt in pairs all return tgt if match name, glob @@ -63,16 +64,16 @@ class BuildObject {target, {dep, @find dep for dep in *target.deps} for name, target in pairs all} @build: (name, upper) => - target=(@find name) or error "No such target: #{name}" + target = (@find name) or error "No such target: #{name}" target\build name, upper __tostring: => "Target #{@name} (#{concat @deps, ', '})" new: (@name, @outs={}, @ins={}, @deps={}, @fn= =>) => - @skip=false + @skip = false error "Duplicate build name #{@name}" if all[@name] - all[@name]=@ + all[@name] = @ build: (name, upper={}) => return if skip[name] @@ -84,43 +85,45 @@ class BuildObject @@build dep, upper for dep in *@deps return unless @shouldbuild name - ins=@ins - outs=@outs + ins = @ins + outs = @outs if @name!=name - ins=[patsubst name, @name, elem for elem in *@ins] - outs=[patsubst name, @name, elem for elem in *@outs] + ins = [patsubst name, @name, elem for elem in *@ins] + outs = [patsubst name, @name, elem for elem in *@outs] print "Building #{@name} as #{name}" else print "Building #{name}" - ok, err=pcall -> + freezecache file for file in *outs + ok, err = pcall -> @.fn ins: ins outs: outs infile: ins[1] outfile: outs[1] name: name + invalidatecache file for file in *outs error "Can't build #{@name}: lua error\n#{err}" unless ok for f in *outs error "Can't build #{@name}: output file #{f} not created" unless exists f - skip[name]=true + skip[name] = true shouldbuild: (name) => return true if args.noskip return true if #@ins==0 or #@outs==0 - ins=if @name!=name + ins = if @name!=name [patsubst name, @name, elem for elem in *@ins] else @ins - itimes=[mtime f for f in *ins] + itimes = [mtime f for f in *ins] for i=1, #@ins error "Can't build #{@name}: missing inputs" unless itimes[i] - outs=if @name!=name + outs = if @name!=name [patsubst name, @name, elem for elem in *@outs] else @outs - otimes=[mtime f for f in *outs] + otimes = [mtime f for f in *outs] for i=1, #@outs return true if not otimes[i] @@ -128,10 +131,10 @@ class BuildObject error "Need Lua >=5.2" if setfenv -targets={} -defaulttarget='all' +targets = {} +defaulttarget = 'all' -buildscope= +buildscope = default: (target) -> defaulttarget=target.name target @@ -139,27 +142,27 @@ buildscope= insert targets, target.name target target: (name, params) -> - tout=flatten params.out - tin=flatten params.in - tdeps=flatten params.deps + tout = flatten params.out + tin = flatten params.in + tdeps = flatten params.deps for f in *flatten params.from insert tin, f insert tdeps, f BuildObject name, tout, tin, tdeps, params.fn :Command -buildscope[k]=fn for k, fn in pairs util +buildscope[k] = fn for k, fn in pairs util setmetatable buildscope, __index: (k) => - global=rawget _G, k + global = rawget _G, k return global if global (...) -> Command k, ... -file=first {'Build.moon', 'Buildfile.moon', 'Build', 'Buildfile'}, exists +file = first {'Build.moon', 'Buildfile.moon', 'Build', 'Buildfile'}, exists error "No Build.moon or Buildfile found" unless file -buildfn=loadwithscope file, buildscope +buildfn = loadwithscope file, buildscope error "Failed to load build function" unless buildfn -ok, err=pcall buildfn +ok, err = pcall buildfn unless ok if err io.stderr\write err, '\n' diff --git a/moonbuild.lua b/moonbuild.lua index e39dbb9..37a1d02 100644 --- a/moonbuild.lua +++ b/moonbuild.lua @@ -1,11 +1,70 @@ do +do +local _ENV = _ENV +package.preload[ "moonbuild.fscache" ] = function( ... ) local arg = _G.arg; +local attributes, dir +do + local _obj_0 = require('lfs') + attributes, dir = _obj_0.attributes, _obj_0.dir +end +local unpack = unpack or table.unpack +local FROZEN +FROZEN = function() end +local makecached +makecached = function(fn) + local cache = { } + local invalidate + invalidate = function(val) + cache[val] = nil + end + local freeze + freeze = function(val) + cache[val] = FROZEN + end + local reset + reset = function() + cache = { } + end + local get + get = function(val) + local cached = cache[val] + if cached ~= FROZEN and cached ~= nil then + return unpack(cached) + end + local ret = { + fn(val) + } + if cached ~= FROZEN then + cache[val] = ret + end + return unpack(ret) + end + return setmetatable({ + get = get, + invalidate = invalidate, + freeze = freeze, + reset = reset + }, { + __call = function(self, val) + return get(val) + end + }) +end +return { + attributes = makecached(attributes), + dir = makecached(dir) +} + +end +end + do local _ENV = _ENV package.preload[ "moonbuild.fsutil" ] = function( ... ) local arg = _G.arg; local dir, attributes do - local _obj_0 = require('lfs') + local _obj_0 = require('moonbuild.fscache') dir, attributes = _obj_0.dir, _obj_0.attributes end local gmatch, match, gsub, sub @@ -13,16 +72,53 @@ do local _obj_0 = string gmatch, match, gsub, sub = _obj_0.gmatch, _obj_0.match, _obj_0.gsub, _obj_0.sub end -local insert, concat +local insert, remove, concat do local _obj_0 = table - insert, concat = _obj_0.insert, _obj_0.concat + insert, remove, concat = _obj_0.insert, _obj_0.remove, _obj_0.concat +end +local normalizepath +normalizepath = function(file) + local parts + do + local _accum_0 = { } + local _len_0 = 1 + for part in gmatch(file, '[^/]+') do + _accum_0[_len_0] = part + _len_0 = _len_0 + 1 + end + parts = _accum_0 + end + local absolute = (sub(file, 1, 1)) == '/' + for i = 1, #parts do + local _continue_0 = false + repeat + if parts[i] == '.' then + remove(parts, i) + i = i - 1 + _continue_0 = true + break + end + if parts[i] == '..' and i ~= 1 then + remove(parts, i) + remove(parts, i - 1) + i = i - 2 + _continue_0 = true + break + end + _continue_0 = true + until true + if not _continue_0 then + break + end + end + return (absolute and '/' or '') .. concat(parts, '/') end local ls ls = function(d) local _accum_0 = { } local _len_0 = 1 - for f in dir(d) do + for f in dir(normalizepath(d)) do if f ~= '.' and f ~= '..' then _accum_0[_len_0] = f _len_0 = _len_0 + 1 @@ -37,7 +133,7 @@ lswithpath = function(d) end local _accum_0 = { } local _len_0 = 1 - for f in dir(d) do + for f in dir(normalizepath(d)) do if f ~= '.' and f ~= '..' then _accum_0[_len_0] = d .. '/' .. f _len_0 = _len_0 + 1 @@ -47,16 +143,16 @@ lswithpath = function(d) end local exists exists = function(f) - return (attributes(f)) ~= nil + return (attributes(normalizepath(f))) ~= nil end local isdir isdir = function(f) - local a = attributes(f) + local a = attributes(normalizepath(f)) return a and a.mode == 'directory' or false end local mtime mtime = function(f) - local a = attributes(f) + local a = attributes(normalizepath(f)) return a and a.modification end local matchglob @@ -152,11 +248,31 @@ wildcard = function(glob) return { } end end +local parentdir +parentdir = function(file) + return normalizepath(file .. '/..') +end +local freezecache +freezecache = function(file) + dir.freeze(file) + dir.freeze(parentdir(file)) + return attributes.invalidate(file) +end +local invalidatecache +invalidatecache = function(file) + dir.invalidate(file) + dir.invalidate(parentdir(file)) + return attributes.invalidate(file) +end return { wildcard = wildcard, exists = exists, isdir = isdir, - mtime = mtime + mtime = mtime, + normalizepath = normalizepath, + parentdir = parentdir, + freezecache = freezecache, + invalidatecache = invalidatecache } end @@ -570,6 +686,11 @@ end local trim trim = require('moonscript.util').trim local util = require('moonbuild.util') +local freezecache, invalidatecache +do + local _obj_0 = require('moonbuild.fsutil') + freezecache, invalidatecache = _obj_0.freezecache, _obj_0.invalidatecache +end local exists, mtime, run, min, max, first, flatten, match, patsubst, sortedpairs exists, mtime, run, min, max, first, flatten, match, patsubst, sortedpairs = util.exists, util.mtime, util.run, util.min, util.max, util.first, util.flatten, util.match, util.patsubst, util.sortedpairs local insert, concat @@ -602,7 +723,7 @@ pcall = function(fn, ...) rewrite = function(err) local trace = debug.traceback('', 2) local trunc = truncate_traceback(trim(trace)) - return rewrite_traceback(trunc, err) + return (rewrite_traceback(trunc, err)) or trace end return xpcall(fn, rewrite, ...) end @@ -721,6 +842,10 @@ do else print("Building " .. tostring(name)) end + for _index_0 = 1, #outs do + local file = outs[_index_0] + freezecache(file) + end local ok, err = pcall(function() return self.fn({ ins = ins, @@ -730,6 +855,10 @@ do name = name }) end) + for _index_0 = 1, #outs do + local file = outs[_index_0] + invalidatecache(file) + end if not (ok) then error("Can't build " .. tostring(self.name) .. ": lua error\n" .. tostring(err)) end diff --git a/moonbuild/fscache.moon b/moonbuild/fscache.moon new file mode 100644 index 0000000..3fd60cd --- /dev/null +++ b/moonbuild/fscache.moon @@ -0,0 +1,33 @@ +import attributes, dir from require 'lfs' +unpack or= table.unpack + +FROZEN = -> + +makecached = (fn) -> + cache = {} + + invalidate = (val) -> + cache[val] = nil + + freeze = (val) -> + cache[val] = FROZEN + + reset = -> + cache = {} + + get = (val) -> + cached = cache[val] + if cached!=FROZEN and cached!=nil + return unpack cached + ret = {fn val} + if cached!=FROZEN + cache[val] = ret + unpack ret + + setmetatable { :get, :invalidate, :freeze, :reset }, + __call: (val) => get val + +{ + attributes: makecached attributes + dir: makecached dir +} diff --git a/moonbuild/fsutil.moon b/moonbuild/fsutil.moon index de42388..b71a2ec 100644 --- a/moonbuild/fsutil.moon +++ b/moonbuild/fsutil.moon @@ -1,25 +1,40 @@ -import dir, attributes from require 'lfs' +import dir, attributes from require 'moonbuild.fscache' import gmatch, match, gsub, sub from string -import insert, concat from table +import insert, remove, concat from table + +normalizepath = (file) -> + parts = [part for part in gmatch file, '[^/]+'] + absolute = (sub file, 1, 1)=='/' + for i=1, #parts + if parts[i]=='.' + remove parts, i + i -= 1 + continue + if parts[i]=='..' and i!=1 + remove parts, i + remove parts, i-1 + i -= 2 + continue + (absolute and '/' or '') .. concat parts, '/' ls = (d) -> - [f for f in dir d when f!='.' and f!='..'] + [f for f in dir normalizepath d when f!='.' and f!='..'] lswithpath = (d) -> if d=='' return ls '.' - [d..'/'..f for f in dir d when f!='.' and f!='..'] + [d..'/'..f for f in dir normalizepath d when f!='.' and f!='..'] exists = (f) -> - (attributes f) != nil + (attributes normalizepath f) != nil isdir = (f) -> - a = attributes f + a = attributes normalizepath f a and a.mode == 'directory' or false mtime = (f) -> - a = attributes f + a = attributes normalizepath f a and a.modification matchglob = (str, glob) -> @@ -81,8 +96,23 @@ wildcard = (glob) -> else return {} +parentdir = (file) -> + normalizepath file..'/..' + +freezecache = (file) -> + dir.freeze file + dir.freeze parentdir file + attributes.invalidate file + +invalidatecache = (file) -> + dir.invalidate file + dir.invalidate parentdir file + attributes.invalidate file + { :wildcard :exists, :isdir :mtime + :normalizepath, :parentdir + :freezecache, :invalidatecache }