fixes to target specific variables

- add a layer for recipe execution
- fix bugs found in new tests
- fix export of an undefined variable

Author: linuxlizard

pymake/pymake.py

@@ -457,13 +457,9 @@ def check_prefixes(s):
     symtable.add_automatic("+", " ".join(rule.prereq_list), rule.get_pos())
     symtable.add_automatic("<", rule.prereq_list[0] if len(rule.prereq_list) else "", rule.get_pos())
 
-    # TODO target specific variables
-
     cmd_s = recipe.eval(symtable)
 #    print("execute_recipe \"%r\"" % cmd_s)
 
-    symtable.pop_layer()
-        
     # Defining Multi-Line Variables.
     # "However, note that using two separate lines means make will invoke the shell twice, running
     # an independent sub-shell for each line. See Section 5.3 [Recipe Execution], page 46."
@@ -529,6 +525,8 @@ def check_prefixes(s):
                 break
             exit_code = 0
 
+    symtable.pop_layer()
+        
     return exit_status["error"] if exit_code else exit_status["success"] 
 
 def execute(makefile, args):
@@ -657,10 +655,18 @@ def execute(makefile, args):
                 # this warning catches where I fail to find an implicit rule
                 logger.warning("I didn't find a recipe to build target=\"%s\"", target)
 
+            # target specific variables
+            if rule.assignment_list:
+                symtable.push_layer()
+                for asn in rule.assignment_list:
+                    asn.eval(symtable)
+
             for recipe in rule.recipe_list:
                 exit_code = execute_recipe(rule, recipe, symtable, args)
                 if exit_code != 0:
                     break
+            if rule.assignment_list:
+                symtable.pop_layer()
             if exit_code != 0:
                 break
 

pymake/symtable.py

@@ -259,6 +259,11 @@ def __init__(self, **kwargs):
 
         self.export_default_value = Export.NOEXPORT
 
+        # A name can be marked export before it's been defined. For example,
+        # export FOO
+        # FOO:=bar
+        self.exported_names = set()
+
         self.warn_undefined = kwargs.get("warn_undefined_variables", False)
 
         self._init_builtins()
@@ -321,6 +326,9 @@ def push_layer(self):
 
     def pop_layer(self):
         # push top, pop top
+        if len(self.layers)==1:
+            # don't allow pop of last layer
+            raise IndexError(1)
         self.layers = self.layers[1:]
 
     def find(self, name, layer_idx=None):
@@ -404,7 +412,12 @@ def add(self, name, value, pos=None):
                     entry = DefaultEntry(name, value, pos)
                 else:
                     entry = FileEntry(name, value, pos)
-                entry.set_export(self.export_default_value)
+
+                # check if the name was exported before a variable was defined
+                if name in self.exported_names:
+                    entry.set_export(Export.EXPLICIT)
+                else:
+                    entry.set_export(self.export_default_value)
 
             self._add_entry(entry)
         else:
@@ -597,8 +610,9 @@ def variables(self, _):
 
         if len(self.layers)==1:
             return " ".join(self.layers[0].keys())
-        raise NotImplementedError("variables")
-#        return " ".join(self.symbols.keys())
+
+        entries = self._get_entries()
+        return " ".join(entries.keys())
 
     def is_defined(self, name):
         # is this varname in our symbol table (or other mechanisms)
@@ -630,11 +644,13 @@ def export(self, name=None):
             return
 
         try:
-            value = self.find(name).set_export(Export.EXPLICIT)
+            self.find(name).set_export(Export.EXPLICIT)
         except KeyError:
             # no such entry
             pass
 
+        self.exported_names.update((name,))
+
     def unexport(self, name=None):
         if name is None:
             # export everything
@@ -647,6 +663,12 @@ def unexport(self, name=None):
             # no such entry
             pass
 
+        try:
+            self.exported_names.remove(name)
+        except KeyError:
+            # no such entry, no big deal
+            pass
+
     def undefine(self, name):
         # support the undefine directive
         if len(self.layers)==1:
@@ -681,14 +703,20 @@ def _unexport_all(self):
         self.export_stop()
 
     def get_exports(self):
+        # Fetch all the exported variables and their values.
+        # Used when launching shell commands so need to eval() the expressions
+        # to get their actual value.
+
         logger.debug("get_exports")
         assert self.env_recursion >= 0
 
-        # cannot eval() during the self.symbols iteration because we could
-        # store something in the symbol table that will throw a "dictionary
-        # changed during iteration" error (for example, .SHELLSTATUS is updated
-        # internally every time a shell is run)
-        entries = self._get_entries(lambda e:e.export )
+        # Separate key:value fetch from .eval() because we could store
+        # something in the symbol table during eval. For example, .SHELLSTATUS
+        # is updated internally every time a shell is run.
+        #
+        # The .eval() during iteration will throw a "dictionary changed during
+        # iteration" error.
+        entries = self._get_entries(lambda e:e.export)
 
         exports = { name:entry.eval(self) for name,entry in entries.items() }
 

tests/target-specific.mk

@@ -11,7 +11,7 @@ BAZ:=baz
 all:CC:=clang
 all:FOO!=echo foo
 all:CFLAGS+=-g
-all:PREREQ:=a.txt
+all:export PREREQ:=a.txt
 
 # PREREQ isn't eval'd so 'all' has no prereqs (the var only applies to the
 # recipe apparently)
@@ -20,7 +20,7 @@ all: $(PREREQ)
 	@printenv CC
 	@printenv FOO
 	@printenv CFLAGS
-	@echo PREREQ=$(PREREQ)
+	@printenv PREREQ
 all:BAR=$(BAZ)
 
 BAZ:=zab

tests/test_export.py

@@ -295,4 +295,18 @@ def test_export_commandline_var_explicit():
     # use explicity assign instead of recursive assign
     run_test(makefile, expect, extra_args=("FOO:=bar",))
 
+def test_export_before_define():
+    makefile="""
+export FOO
+FOO:=foo
+@:; printenv FOO
+"""
+    run.simple_test(makefile)
+
+def test_unexpert_unknown():
+    makefile="""
+unexport FOO
+@:;@:
+"""
+    run.simple_test(makefile)
 

tests/test_makefile.py

@@ -113,6 +113,7 @@ def _run_pymake(infilename):
     "ignoreandsilent.mk",
     "rule_without_target.mk",
     "shellstatus.mk",
+    "target-specific.mk",
 
     # FIXME fails with GNU Make 4.3
 #    "env_recursion.mk",

tests/test_symtable.py

@@ -32,80 +32,75 @@ def test_recursively_expanded():
     value = symbol_table.fetch("CFLAGS")
     assert value=="-g -Wall", value
 
-@pytest.mark.skip(reason="FIXME push/pop")
-def test_simple_push_pop():
+def test_simple_layer():
     symbol_table = symtable.SymbolTable()
-    symbol_table.add("target", ["abcdefghijklmnopqrstuvwxyz"])
+    symbol_table.add("target", "abcdefghijklmnopqrstuvwxyz")
 
-    symbol_table.push("target")
-    symbol_table.add("target", ["12345"])
+    symbol_table.push_layer()
+    symbol_table.add("target", "12345")
     value = symbol_table.fetch("target")
-    assert value == ["12345"]
+    assert value == "12345"
 
-    symbol_table.pop("target")
+    symbol_table.pop_layer()
     value = symbol_table.fetch("target")
-    assert value == ["abcdefghijklmnopqrstuvwxyz"]
+    assert value == "abcdefghijklmnopqrstuvwxyz"
 
-@pytest.mark.skip(reason="FIXME push/pop")
 def test_push_push_pop_pop():
     symbol_table = symtable.SymbolTable()
-    symbol_table.add("target", ["abcdefghijklmnopqrstuvwxyz"])
+    symbol_table.add("target", "abcdefghijklmnopqrstuvwxyz")
 
-    symbol_table.push("target")
-    symbol_table.add("target", ["12345"])
+    symbol_table.push_layer()
+    symbol_table.add("target", "12345")
 
-    symbol_table.push("target")
-    symbol_table.add("target", ["67890"])
+    symbol_table.push_layer()
+    symbol_table.add("target", "67890")
 
     value = symbol_table.fetch("target")
-    assert value == ["67890"]
+    assert value == "67890"
 
-    symbol_table.pop("target")
+    symbol_table.pop_layer()
     value = symbol_table.fetch("target")
-    assert value == ["12345"]
+    assert value == "12345"
 
-    symbol_table.pop("target")
+    symbol_table.pop_layer()
     value = symbol_table.fetch("target")
-    assert value == ["abcdefghijklmnopqrstuvwxyz"]
+    assert value == "abcdefghijklmnopqrstuvwxyz"
 
-@pytest.mark.skip(reason="FIXME push/pop")
 def test_push_pop_undefined():
     # "If var was undefined before the foreach function call, it is undefined after the call."
     symbol_table = symtable.SymbolTable()
 
-    symbol_table.push("target")
-    symbol_table.add("target", ["12345"])
+    symbol_table.push_layer()
+    symbol_table.add("target", "12345")
     value = symbol_table.fetch("target")
-    assert value == ["12345"]
+    assert value == "12345"
 
-    symbol_table.pop("target")
+    symbol_table.pop_layer()
     value = symbol_table.fetch("target")
     assert value==""
 
-@pytest.mark.skip(reason="FIXME push/pop")
 def test_push_pop_pop():
     # too many pops
     symbol_table = symtable.SymbolTable()
-    symbol_table.add("target", ["abcdefghijklmnopqrstuvwxyz"])
+    symbol_table.add("target", "abcdefghijklmnopqrstuvwxyz")
 
-    symbol_table.push("target")
-    symbol_table.add("target", ["12345"])
+    symbol_table.push_layer()
+    symbol_table.add("target", "12345")
     value = symbol_table.fetch("target")
-    assert value == ["12345"]
+    assert value == "12345"
 
-    symbol_table.pop("target")
+    symbol_table.pop_layer()
     value = symbol_table.fetch("target")
-    assert value == ["abcdefghijklmnopqrstuvwxyz"]
+    assert value == "abcdefghijklmnopqrstuvwxyz"
 
     try:
-        symbol_table.pop("target")
+        symbol_table.pop_layer()
     except IndexError:
         pass
     else:
         # expected IndexError
         assert 0
 
-@pytest.mark.skip(reason="FIXME push/pop")
 def test_env_var():
     # environment variables should act like regular vars
     symbol_table = symtable.SymbolTable()
@@ -115,12 +110,12 @@ def test_env_var():
     assert save_path
     assert symbol_table.origin("PATH") == "environment"
 
-    symbol_table.push("PATH")
+    symbol_table.push_layer()
     symbol_table.add("PATH", "a:b:c:")
     value = symbol_table.fetch("PATH")
     assert value == "a:b:c:"
 
-    symbol_table.pop("PATH")
+    symbol_table.pop_layer()
 
     path = symbol_table.fetch("PATH")
     assert path==save_path
@@ -144,6 +139,13 @@ def test_is_defined():
     symbol_table.add("FOO", "BAR")
     assert symbol_table.is_defined("FOO")
 
+    assert not symbol_table.is_defined("DAVE")
+    symbol_table.push_layer()
+    symbol_table.add("DAVE","dave")
+    assert symbol_table.is_defined("DAVE")
+    symbol_table.pop_layer()
+    assert not symbol_table.is_defined("DAVE")
+
 def test_maybe_add():
     symbol_table = symtable.SymbolTable()
 
@@ -286,5 +288,58 @@ def test_command_line_multiple_var():
     assert symbol_table.origin("FOO") == "command line"
 
 def test_layers():
-    pass
+    symbol_table = symtable.SymbolTable()
+    symbol_table.add("FOO", Literal("foo"))
+    symbol_table.push_layer()
+    value = symbol_table.fetch("FOO")
+    assert value == "foo"  # unchanged
+
+def test_layers_variables():
+    symbol_table = symtable.SymbolTable()
+
+    symbol_table.add("FOO", Literal("foo"))
+    varlist = symbol_table.variables(None)
+    assert "FOO" in varlist
+
+    symbol_table.push_layer()
+    varlist = symbol_table.variables(None)
+    assert "FOO" in varlist
+
+@pytest.mark.skip("undefine FIXME")
+def test_layers_undefine():
+    symbol_table = symtable.SymbolTable()
+
+    symbol_table.add("FOO", Literal("foo"))
+    symbol_table.push_layer()
+    symbol_table.add("FOO", Literal("bar"))
+
+    symbol_table.undefine("FOO")
+    varlist = symbol_table.variables(None)
+    assert "FOO" not in varlist
+    symbol_table.pop_layer()
+    varlist = symbol_table.variables(None)
+    assert "FOO" in varlist
+
+@pytest.mark.skip("foo")
+def test_layer_append():
+    symbol_table = symtable.SymbolTable()
+
+    symbol_table.add("FOO", Literal("foo"))
+    symbol_table.push_layer()
+    symbol_table.append("FOO", Literal("bar"))
+    value = symbol_table.fetch("FOO")
+#    assert value=="foobar"
+    symbol_table.pop_layer()
+    value = symbol_table.fetch("FOO")
+#    assert value=="foo"
+
+def test_global_export_layers():
+    makefile="""
+export
+
+all:FOO:=bar
+all:
+	printenv FOO
+"""
+    run.simple_test(makefile)