Merge pull request #770 from davidgiven/ab

Update ab.

.github/workflows/ccpp.yml

@@ -43,7 +43,7 @@ jobs:
     - name: Upload build artifacts
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ github.event.repository.name }}.${{ github.sha }}
+        name: ${{ github.event.repository.name }}.${{ github.sha }}.fluxengine.pkg
         path: fluxengine/FluxEngine.pkg
 
   build-windows:
@@ -56,6 +56,7 @@ jobs:
         unzip fedora.msixbundle Fedora-Remix-for-WSL-SL_40.1.0.0_x64.msix
         unzip Fedora-Remix-for-WSL-SL_40.1.0.0_x64.msix install.tar.gz
         wsl --update
+        wsl --set-default-version 2
         wsl --import fedora fedora install.tar.gz
         wsl --set-default fedora
         wsl sh -c 'dnf -y install https://github.com/rpmsphere/noarch/raw/master/r/rpmsphere-release-38-1.noarch.rpm'
@@ -93,5 +94,5 @@ jobs:
     - name: Upload build artifacts
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ github.event.repository.name }}.${{ github.sha }}
+        name: ${{ github.event.repository.name }}.${{ github.sha }}.windows.zip
         path: fluxengine/fluxengine-windows.zip

.github/workflows/release.yml

@@ -20,6 +20,7 @@ jobs:
         unzip fedora.msixbundle Fedora-Remix-for-WSL-SL_39.0.1.0_x64.msix
         unzip Fedora-Remix-for-WSL-SL_39.0.1.0_x64.msix install.tar.gz
         wsl --update
+        wsl --set-default-version 2
         wsl --import fedora fedora install.tar.gz
         wsl --set-default fedora
         wsl sh -c 'dnf -y install https://github.com/rpmsphere/noarch/raw/master/r/rpmsphere-release-38-1.noarch.rpm'

build/_progress.py

@@ -0,0 +1,5 @@
+import sys
+
+(_, current, max) = sys.argv
+percent = int(100 * float(current) / float(max))
+print(f"[{percent:>3}%]")

build/ab.mk

@@ -28,6 +28,12 @@ ifeq ($(OS), Windows_NT)
 endif
 EXT ?=
 
+ifeq ($(PROGRESSINFO),)
+rulecount := $(shell $(MAKE) --no-print-directory -q $(OBJ)/build.mk PROGRESSINFO=1 && $(MAKE) -n $(MAKECMDGOALS) PROGRESSINFO=XXXPROGRESSINFOXXX | grep XXXPROGRESSINFOXXX | wc -l)
+ruleindex := 1
+PROGRESSINFO = "$(shell $(PYTHON) build/_progress.py $(ruleindex) $(rulecount))$(eval ruleindex := $(shell expr $(ruleindex) + 1))"
+endif
+
 include $(OBJ)/build.mk
 
 MAKEFLAGS += -r -j$(shell nproc)
@@ -47,8 +53,8 @@ clean::
 
 export PYTHONHASHSEED = 1
 build-files = $(shell find . -name 'build.py') $(wildcard build/*.py) $(wildcard config.py)
-$(OBJ)/build.mk: Makefile $(build-files)
+$(OBJ)/build.mk: Makefile $(build-files) build/ab.mk
 	@echo "AB"
 	@mkdir -p $(OBJ)
-	$(hide) $(PYTHON) -X pycache_prefix=$(OBJ) build/ab.py -o $@ build.py \
+	$(hide) $(PYTHON) -X pycache_prefix=$(OBJ)/__pycache__ build/ab.py -o $@ build.py \
 		|| rm -f $@

build/ab.py

@@ -16,6 +16,7 @@
 import inspect
 import string
 import sys
+import hashlib
 
 verbose = False
 quiet = False
@@ -131,6 +132,12 @@ def wrapper(*, name=None, replaces=None, **kwargs):
     return wrapper
 
 
+def _isiterable(xs):
+    return isinstance(xs, Iterable) and not isinstance(
+        xs, (str, bytes, bytearray)
+    )
+
+
 class Target:
     def __init__(self, cwd, name):
         if verbose:
@@ -147,6 +154,9 @@ def __init__(self, cwd, name):
     def __eq__(self, other):
         return self.name is other.name
 
+    def __lt__(self, other):
+        return self.name < other.name
+
     def __hash__(self):
         return id(self)
 
@@ -166,7 +176,7 @@ def format_field(self, value, format_spec):
                     return ""
                 if type(value) == str:
                     return value
-                if isinstance(value, (set, tuple)):
+                if _isiterable(value):
                     value = list(value)
                 if type(value) != list:
                     value = [value]
@@ -210,9 +220,23 @@ def materialise(self, replacing=False):
             # Actually call the callback.
 
             cwdStack.append(self.cwd)
-            self.callback(
-                **{k: self.args[k] for k in self.binding.arguments.keys()}
-            )
+            if "kwargs" in self.binding.arguments.keys():
+                # If the caller wants kwargs, return all arguments except the standard ones.
+                cbargs = {
+                    k: v for k, v in self.args.items() if k not in {"dir"}
+                }
+            else:
+                # Otherwise, just call the callback with the ones it asks for.
+                cbargs = {}
+                for k in self.binding.arguments.keys():
+                    if k != "kwargs":
+                        try:
+                            cbargs[k] = self.args[k]
+                        except KeyError:
+                            error(
+                                f"invocation of {self} failed because {k} isn't an argument"
+                            )
+            self.callback(**cbargs)
             cwdStack.pop()
         except BaseException as e:
             print(f"Error materialising {self}: {self.callback}")
@@ -293,7 +317,12 @@ def targetof(value, cwd=None):
             # Load the new build file.
 
             path = join(path, "build.py")
-            loadbuildfile(path)
+            try:
+                loadbuildfile(path)
+            except ModuleNotFoundError:
+                error(
+                    f"no such build file '{path}' while trying to resolve '{value}'"
+                )
             assert (
                 value in targets
             ), f"build file at '{path}' doesn't contain '+{target}' when trying to resolve '{value}'"
@@ -308,9 +337,7 @@ class Targets:
     def convert(value, target):
         if not value:
             return []
-        assert isinstance(
-            value, (list, tuple)
-        ), "cannot convert non-list to Targets"
+        assert _isiterable(value), "cannot convert non-list to Targets"
         return [target.targetof(x) for x in flatten(value)]
 
 
@@ -334,7 +361,7 @@ def loadbuildfile(filename):
 def flatten(items):
     def generate(xs):
         for x in xs:
-            if isinstance(x, Iterable) and not isinstance(x, (str, bytes)):
+            if _isiterable(x):
                 yield from generate(x)
             else:
                 yield x
@@ -343,15 +370,13 @@ def generate(xs):
 
 
 def targetnamesof(items):
-    if not isinstance(items, (list, tuple, set)):
-        error("argument of filenamesof is not a list/tuple/set")
+    assert _isiterable(items), "argument of filenamesof is not a collection"
 
     return [t.name for t in items]
 
 
 def filenamesof(items):
-    if not isinstance(items, (list, tuple, set)):
-        error("argument of filenamesof is not a list/tuple/set")
+    assert _isiterable(items), "argument of filenamesof is not a collection"
 
     def generate(xs):
         for x in xs:
@@ -371,9 +396,12 @@ def filenameof(x):
     return xs[0]
 
 
-def emit(*args):
-    outputFp.write(" ".join(args))
-    outputFp.write("\n")
+def emit(*args, into=None):
+    s = " ".join(args) + "\n"
+    if into is not None:
+        into += [s]
+    else:
+        outputFp.write(s)
 
 
 def emit_rule(name, ins, outs, cmds=[], label=None):
@@ -382,26 +410,35 @@ def emit_rule(name, ins, outs, cmds=[], label=None):
     nonobjs = [f for f in fouts if not f.startswith("$(OBJ)")]
 
     emit("")
+
+    lines = []
     if nonobjs:
-        emit("clean::")
-        emit("\t$(hide) rm -f", *nonobjs)
+        emit("clean::", into=lines)
+        emit("\t$(hide) rm -f", *nonobjs, into=lines)
 
-    emit(".PHONY:", name)
+    emit(".PHONY:", name, into=lines)
     if outs:
-        emit(name, ":", *fouts)
-        if cmds:
-            emit(*fouts, "&:", *fins)
-        else:
-            emit(*fouts, ":", *fins)
+        emit(name, ":", *fouts, into=lines)
+        emit(*fouts, "&:" if len(fouts) > 1 else ":", *fins, "\x01", into=lines)
 
         if label:
-            emit("\t$(hide)", "$(ECHO)", label)
+            emit("\t$(hide)", "$(ECHO) $(PROGRESSINFO) ", label, into=lines)
         for c in cmds:
-            emit("\t$(hide)", c)
+            emit("\t$(hide)", c, into=lines)
     else:
         assert len(cmds) == 0, "rules with no outputs cannot have commands"
-        emit(name, ":", *fins)
+        emit(name, ":", *fins, into=lines)
+
+    cmd = "".join(lines)
+    hash = hashlib.sha1(bytes(cmd, "utf-8")).hexdigest()
 
+    outputFp.write(cmd.replace("\x01", f"$(OBJ)/.hashes/{hash}"))
+
+    if outs:
+        emit(f"$(OBJ)/.hashes/{hash}:")
+        emit(
+            f"\t$(hide) mkdir -p $(OBJ)/.hashes && touch $(OBJ)/.hashes/{hash}"
+        )
     emit("")
 
 

build/utils.py

@@ -8,8 +8,8 @@
     error,
     simplerule,
 )
-from os.path import relpath, splitext, join, basename
-from glob import glob
+from os.path import relpath, splitext, join, basename, isfile
+from glob import iglob
 import fnmatch
 import itertools
 
@@ -30,7 +30,7 @@ def collectattrs(*, targets, name, initial=[]):
     s = set(initial)
     for a in [t.args.get(name, []) for t in targets]:
         s.update(a)
-    return list(s)
+    return sorted(list(s))
 
 
 def itemsof(pattern, root=None, cwd=None):
@@ -43,9 +43,10 @@ def itemsof(pattern, root=None, cwd=None):
     root = join(cwd, root)
 
     result = {}
-    for f in glob(pattern, recursive=True):
+    for f in iglob(pattern, recursive=True):
         try:
-            result[relpath(f, root)] = f
+            if isfile(f):
+                result[relpath(f, root)] = f
         except ValueError:
             error(f"file '{f}' is not in root '{root}'")
     return result