Update ab.

Author: davidgiven

build/_sandbox.py

@@ -26,7 +26,7 @@ def main():
                 print("link", sf)
             os.makedirs(dirname(sf), exist_ok=True)
             try:
-                os.link(abspath(f), sf)
+                os.symlink(abspath(f), sf)
             except PermissionError:
                 shutil.copy(f, sf)
 
@@ -38,6 +38,11 @@ def main():
             df = dirname(f)
             if df:
                 os.makedirs(df, exist_ok=True)
+
+            try:
+                os.remove(f)
+            except FileNotFoundError:
+                pass
             os.rename(sf, f)
 
 

build/ab.mk

@@ -1,23 +1,28 @@
 MAKENOT4 := $(if $(findstring 3.9999, $(lastword $(sort 3.9999 $(MAKE_VERSION)))),yes,no)
-MAKE4.3 := $(if $(findstring 4.3, $(firstword $(sort 4.3 $(MAKE_VERSION)))),yes,no)
-MAKE4.1 := $(if $(findstring no_no,$(MAKENOT4)_$(MAKE4.3)),yes,no)
 
-ifeq ($(MAKENOT3),yes)
+ifeq ($(MAKENOT4),yes)
 $(error You need GNU Make 4.x for this (if you're on OSX, use gmake).)
 endif
 
 OBJ ?= .obj
 PYTHON ?= python3
-CC ?= gcc
-CXX ?= g++
-AR ?= ar
-CFLAGS ?= -g -Og
-LDFLAGS ?= -g
 PKG_CONFIG ?= pkg-config
 HOST_PKG_CONFIG ?= $(PKG_CONFIG)
 ECHO ?= echo
 CP ?= cp
 
+HOSTCC ?= gcc
+HOSTCXX ?= g++
+HOSTAR ?= ar
+HOSTCFLAGS ?= -g -Og
+HOSTLDFLAGS ?= -g
+
+CC ?= $(HOSTCC)
+CXX ?= $(HOSTCXX)
+AR ?= $(HOSTAR)
+CFLAGS ?= $(HOSTCFLAGS)
+LDFLAGS ?= $(HOSTLDFLAGS)
+
 export PKG_CONFIG
 export HOST_PKG_CONFIG
 
@@ -31,6 +36,11 @@ else
 	endif
 endif
 
+# If enabled, shows a nice display of how far through the build you are. This
+# doubles Make startup time. Also, on Make 4.3 and above, rebuilds don't show
+# correct progress information.
+AB_ENABLE_PROGRESS_INFO ?= true
+
 WINDOWS := no
 OSX := no
 LINUX := no
@@ -53,13 +63,17 @@ EXT ?=
 
 CWD=$(shell pwd)
 
-ifeq ($(PROGRESSINFO),)
-# The first make invocation here has to have its output discarded or else it
-# produces spurious 'Leaving directory' messages... don't know why.
-rulecount := $(strip $(shell $(MAKE) --no-print-directory -q $(OBJ)/build.mk PROGRESSINFO=1 > /dev/null \
-	&& $(MAKE) --no-print-directory -n $(MAKECMDGOALS) PROGRESSINFO=XXXPROGRESSINFOXXX | grep XXXPROGRESSINFOXXX | wc -l))
-ruleindex := 1
-PROGRESSINFO = "[$(ruleindex)/$(rulecount)]$(eval ruleindex := $(shell expr $(ruleindex) + 1))"
+ifeq ($(AB_ENABLE_PROGRESS_INFO),true)
+	ifeq ($(PROGRESSINFO),)
+	# The first make invocation here has to have its output discarded or else it
+	# produces spurious 'Leaving directory' messages... don't know why.
+	rulecount := $(strip $(shell $(MAKE) --no-print-directory -q $(OBJ)/build.mk PROGRESSINFO=1 > /dev/null \
+		&& $(MAKE) --no-print-directory -n $(MAKECMDGOALS) PROGRESSINFO=XXXPROGRESSINFOXXX | grep XXXPROGRESSINFOXXX | wc -l))
+	ruleindex := 1
+	PROGRESSINFO = "[$(ruleindex)/$(rulecount)]$(eval ruleindex := $(shell expr $(ruleindex) + 1)) "
+	endif
+else
+	PROGRESSINFO = ""
 endif
 
 PKG_CONFIG_HASHES = $(OBJ)/.pkg-config-hashes/target-$(word 1, $(shell $(PKG_CONFIG) --list-all | md5sum))

build/ab.py

@@ -6,7 +6,6 @@
 from copy import copy
 import functools
 import importlib
-import importlib.abc
 import importlib.util
 from importlib.machinery import (
     SourceFileLoader,
@@ -21,13 +20,17 @@
 import ast
 from collections import namedtuple
 
+VERBOSE_MK_FILE = False
+
 verbose = False
 quiet = False
 cwdStack = [""]
 targets = {}
 unmaterialisedTargets = {}  # dict, not set, to get consistent ordering
 materialisingStack = []
 defaultGlobals = {}
+globalId = 1
+wordCache = {}
 
 RE_FORMAT_SPEC = re.compile(
     r"(?:(?P<fill>[\s\S])?(?P<align>[<>=^]))?"
@@ -157,7 +160,8 @@ def wrapper(*, name=None, replaces=None, **kwargs):
         t.callback = func
         t.traits.add(func.__name__)
         if "args" in kwargs:
-            t.args.update(kwargs["args"])
+            t.explicit_args = kwargs["args"]
+            t.args.update(t.explicit_args)
             del kwargs["args"]
         if "traits" in kwargs:
             t.traits |= kwargs["traits"]
@@ -406,8 +410,17 @@ def _removesuffix(self, suffix):
 
 
 def loadbuildfile(filename):
-    filename = _removesuffix(filename.replace("/", "."), ".py")
-    builtins.__import__(filename)
+    modulename = _removesuffix(filename.replace("/", "."), ".py")
+    if modulename not in sys.modules:
+        spec = importlib.util.spec_from_file_location(
+            name=modulename,
+            location=filename,
+            loader=BuildFileLoaderImpl(fullname=modulename, path=filename),
+            submodule_search_locations=[],
+        )
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[modulename] = module
+        spec.loader.exec_module(module)
 
 
 def flatten(items):
@@ -433,6 +446,7 @@ def filenamesof(items):
     def generate(xs):
         for x in xs:
             if isinstance(x, Target):
+                x.materialise()
                 yield from generate(x.outs)
             else:
                 yield x
@@ -458,63 +472,67 @@ def emit(*args, into=None):
 
 def emit_rule(self, ins, outs, cmds=[], label=None):
     name = self.name
-    fins = set(filenamesof(ins))
+    fins_list = filenamesof(ins)
+    fins = set(fins_list)
     fouts = filenamesof(outs)
     nonobjs = [f for f in fouts if not f.startswith("$(OBJ)")]
 
     emit("")
+    if VERBOSE_MK_FILE:
+        for k, v in self.args.items():
+            emit(f"# {k} = {v}")
 
     lines = []
     if nonobjs:
         emit("clean::", into=lines)
         emit("\t$(hide) rm -f", *nonobjs, into=lines)
 
+    hashable = cmds + fins_list + fouts
+    hash = hashlib.sha1(bytes("\n".join(hashable), "utf-8")).hexdigest()
+    hashfile = join(self.dir, f"hash_{hash}")
+
+    global globalId
     emit(".PHONY:", name, into=lines)
     if outs:
-        emit(name, ":", *fouts, into=lines)
-        if len(fouts) == 1:
-            emit(*fouts, ":", *fins, "\x01", into=lines)
-        else:
-            emit("ifeq ($(MAKE4.3),yes)", into=lines)
-            emit(*fouts, "&:", *fins, "\x01", into=lines)
-            emit("else", into=lines)
-            emit(*(fouts[1:]), ":", fouts[0], into=lines)
-            emit(fouts[0], ":", *fins, "\x01", into=lines)
-            emit("endif", into=lines)
+        outsn = globalId
+        globalId = globalId + 1
+        insn = globalId
+        globalId = globalId + 1
+
+        emit(f"OUTS_{outsn}", "=", *fouts, into=lines)
+        emit(f"INS_{insn}", "=", *fins, into=lines)
+        emit(name, ":", f"$(OUTS_{outsn})")
+        emit(hashfile, ":")
+        emit(f"\t@mkdir -p {self.dir}")
+        emit(f"\t@touch {hashfile}")
+        emit(f"$(OUTS_{outsn})", "&:",f"$(INS_{insn})", hashfile, into=lines)
 
         if label:
-            emit("\t$(hide)", "$(ECHO) $(PROGRESSINFO)", label, into=lines)
+            emit("\t$(hide)", "$(ECHO) $(PROGRESSINFO)" + label, into=lines)
 
         sandbox = join(self.dir, "sandbox")
         emit("\t$(hide)", f"rm -rf {sandbox}", into=lines)
         emit(
             "\t$(hide)",
-            f"$(PYTHON) build/_sandbox.py --link -s {sandbox}",
-            *fins,
+            "$(PYTHON) build/_sandbox.py --link -s",
+            sandbox,
+            f"$(INS_{insn})",
             into=lines,
         )
         for c in cmds:
             emit(f"\t$(hide) cd {sandbox} && (", c, ")", into=lines)
         emit(
             "\t$(hide)",
-            f"$(PYTHON) build/_sandbox.py --export -s {sandbox}",
-            *fouts,
+            "$(PYTHON) build/_sandbox.py --export -s",
+            sandbox,
+            f"$(OUTS_{outsn})",
             into=lines,
         )
     else:
         assert len(cmds) == 0, "rules with no outputs cannot have commands"
         emit(name, ":", *fins, into=lines)
 
-    cmd = "".join(lines)
-    hash = hashlib.sha1(bytes(cmd, "utf-8")).hexdigest()
-
-    outputFp.write(cmd.replace("\x01", f"$(OBJ)/.hashes/{hash}"))
-
-    if outs:
-        emit(f"$(OBJ)/.hashes/{hash}:")
-        emit(
-            f"\t$(hide) mkdir -p $(OBJ)/.hashes && touch $(OBJ)/.hashes/{hash}"
-        )
+    outputFp.write("".join(lines))
     emit("")
 
 
@@ -578,13 +596,12 @@ def export(self, name=None, items: TargetsMap = {}, deps: Targets = []):
         )
         subrule.materialise()
 
-    simplerule(
-        replaces=self,
-        ins=outs + deps,
-        outs=["=sentinel"],
-        commands=["touch $[outs[0]]"],
-        label="EXPORT",
-    )
+    self.ins = []
+    self.outs = deps + outs
+
+    emit("")
+    emit(".PHONY:", name)
+    emit(name, ":", *filenamesof(outs + deps))
 
 
 def main():