Allow postprocessing filters to be disabled

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 * Validation of component slug ([#153])
 * `component compile` now applies postprocessing filters ([#154])
+* Option to disable postprocessing filters ([#155])
 
 ## [v0.2.2]
 
@@ -145,3 +146,4 @@ Initial implementation
 [#151]: https://github.com/projectsyn/commodore/pull/151
 [#153]: https://github.com/projectsyn/commodore/pull/153
 [#154]: https://github.com/projectsyn/commodore/pull/154
+[#155]: https://github.com/projectsyn/commodore/pull/155

commodore/postprocess/__init__.py

@@ -6,6 +6,7 @@
 
 from .jsonnet import run_jsonnet_filter
 from .builtin_filters import run_builtin_filter
+from .inventory import resolve_inventory_vars, InventoryError
 
 
 def postprocess_components(config, inventory, target, components):
@@ -20,11 +21,23 @@ def postprocess_components(config, inventory, target, components):
                 click.echo(f" > {cn}...")
             filters = yaml_load(filterdir / 'filters.yml')
             for f in filters['filters']:
-                # old-style filters are always 'jsonnet'
+                # Resolve any inventory references in filter definition
+                try:
+                    f = resolve_inventory_vars(inventory, f)
+                except InventoryError as e:
+                    raise click.ClickException(
+                        f"Failed to resolve variables for old-style filter: {e}") from e
+
+                # filters without 'type' are always 'jsonnet'
                 if 'type' not in f:
                     click.secho('   > [WARN] component uses old-style postprocess filters',
                                 fg='yellow')
                     f['type'] = 'jsonnet'
+                # Filters which aren't explicitly disabled are always enabled
+                if 'enabled' in f and not f['enabled']:
+                    click.secho('   > Skipping disabled filter' +
+                                f" {f['filter']} on path {f['path']}")
+                    continue
                 if f['type'] == 'jsonnet':
                     run_jsonnet_filter(inventory, cn, target, filterdir, f)
                 elif f['type'] == 'builtin':

commodore/postprocess/builtin_filters.py

@@ -1,5 +1,4 @@
 import json
-import re
 
 from pathlib import Path as P
 
@@ -35,41 +34,9 @@ def _builtin_filter_helm_namespace(inv, component, target, path, **kwargs):
 }
 
 
-class InventoryError(Exception):
-    pass
-
-
-def _resolve_var(inv, m):
-    var = m.group(1)
-    invpath = var.split(':')
-    val = inv['parameters']
-    for elem in invpath:
-        val = val.get(elem, None)
-        if val is None:
-            raise InventoryError(f"Unable to resolve inventory reference {var}")
-    return val
-
-
-INV_REF = re.compile(r'\$\{([^}]+)\}')
-
-
-def _resolve_inventory_vars(inv, args):
-    resolved = {}
-    for k, v in args.items():
-        if isinstance(v, str):
-            resolved[k] = INV_REF.sub(lambda m: _resolve_var(inv, m), v)
-        else:
-            resolved[k] = v
-    return resolved
-
-
 def run_builtin_filter(inv, component, target, f):
     fname = f['filter']
     if fname not in _builtin_filters:
         click.secho(f"   > [ERR ] Unknown builtin filter {fname}", fg='red')
         return
-    try:
-        filterargs = _resolve_inventory_vars(inv, f['filterargs'])
-    except InventoryError as e:
-        raise click.ClickException(f"Failure in builtin filter: {e}") from e
-    _builtin_filters[fname](inv, component, target, f['path'], **filterargs)
+    _builtin_filters[fname](inv, component, target, f['path'], **f['filterargs'])

commodore/postprocess/inventory.py

@@ -0,0 +1,45 @@
+import re
+
+
+class InventoryError(Exception):
+    pass
+
+
+INV_REF = re.compile(r'\$\{([^}]+)\}')
+
+
+def _resolve_var(inv, m):
+    var = m.group(1)
+    invpath = var.split(':')
+    val = inv['parameters']
+    for elem in invpath:
+        val = val.get(elem, None)
+        if val is None:
+            raise InventoryError(f"Unable to resolve inventory reference {var}")
+    return val
+
+
+def resolve_inventory_vars(inv, args):
+    """
+    Recursively resolve reclass references in `args`.
+    """
+    resolved = {}
+    for k, v in args.items():
+        if isinstance(v, list):
+            resolved[k] = map(lambda e: resolve_inventory_vars(inv, e), v)
+        elif isinstance(v, dict):
+            resolved[k] = resolve_inventory_vars(inv, v)
+        elif isinstance(v, str):
+            try:
+                resolved[k] = INV_REF.sub(lambda m: _resolve_var(inv, m), v)
+            except TypeError as e:
+                if v.startswith('${') and v.endswith('}'):
+                    m = INV_REF.match(v)
+                    if m is None:
+                        raise InventoryError(f"Error replacing reference: {e}") from e
+                    resolved[k] = _resolve_var(inv, m)
+                else:
+                    raise e
+        else:
+            resolved[k] = v
+    return resolved

tests/test_postprocess.py

@@ -0,0 +1,133 @@
+"""
+Tests for postprocessing
+"""
+import os
+import yaml
+from commodore.config import Config, Component
+from commodore.postprocess import postprocess_components
+from git import Repo
+from test_component_new import test_run_component_new_command
+
+
+def _make_ns_filter(ns, enabled=None):
+    filter = {
+        'filters': [{
+            'path': 'test',
+            'type': 'builtin',
+            'filter': 'helm_namespace',
+            'filterargs': {
+                'namespace': ns,
+            }
+        }]
+    }
+    if enabled is not None:
+        filter['filters'][0]['enabled'] = enabled
+    return filter
+
+
+def _setup(tmp_path, filter):
+    os.chdir(tmp_path)
+
+    test_run_component_new_command(tmp_path=tmp_path)
+
+    target = 'target'
+    targetdir = tmp_path / 'compiled' / target / 'test'
+    os.makedirs(targetdir, exist_ok=True)
+    testf = targetdir / 'object.yaml'
+    with open(testf, 'w') as objf:
+        obj = {
+            'metadata': {
+                'name': 'test',
+                'namespace': 'untouched',
+            },
+            'kind': 'Secret',
+            'apiVersion': 'v1',
+            'stringData': {
+                'content': 'verysecret',
+            },
+        }
+        yaml.dump(obj, objf)
+    with open(tmp_path / 'dependencies' / 'test-component' / 'postprocess' /
+              'filters.yml', 'w') as filterf:
+        yaml.dump(filter, filterf)
+
+    config = Config()
+    component = Component('test-component',
+                          Repo(tmp_path / 'dependencies' / 'test-component'),
+                          'https://fake.repo.url',
+                          'master')
+    inventory = {
+        'classes': {
+            'defaults.test-component',
+            'global.common',
+            'components.test-component',
+        },
+        'parameters': {
+            'test_component': {
+                'namespace': 'syn-test-component',
+            },
+        },
+    }
+    return testf, config, inventory, target, {'test-component': component}
+
+
+def test_postprocess_components(tmp_path, capsys):
+    filter = _make_ns_filter('myns')
+    testf, config, inventory, target, components = _setup(tmp_path, filter)
+    postprocess_components(config, inventory, target, components)
+    assert testf.exists()
+    with open(testf) as objf:
+        obj = yaml.safe_load(objf)
+        assert obj['metadata']['namespace'] == 'myns'
+
+
+def test_postprocess_components_enabled(tmp_path, capsys):
+    filter = _make_ns_filter('myns', enabled=True)
+    testf, config, inventory, target, components = _setup(tmp_path, filter)
+    postprocess_components(config, inventory, target, components)
+    assert testf.exists()
+    with open(testf) as objf:
+        obj = yaml.safe_load(objf)
+        assert obj['metadata']['namespace'] == 'myns'
+
+
+def test_postprocess_components_disabled(tmp_path, capsys):
+    filter = _make_ns_filter('myns', enabled=False)
+    testf, config, inventory, target, components = _setup(tmp_path, filter)
+    postprocess_components(config, inventory, target, components)
+    assert testf.exists()
+    with open(testf) as objf:
+        obj = yaml.safe_load(objf)
+        assert obj['metadata']['namespace'] == 'untouched'
+    captured = capsys.readouterr()
+    assert "Skipping disabled filter" in captured.out
+
+
+def test_postprocess_components_enabledref(tmp_path, capsys):
+    filter = _make_ns_filter('myns',
+                             enabled='${test_component:filter:enabled}')
+    testf, config, inventory, target, components = _setup(tmp_path, filter)
+    inventory['parameters']['test_component']['filter'] = {
+        'enabled': True,
+    }
+    postprocess_components(config, inventory, target, components)
+    assert testf.exists()
+    with open(testf) as objf:
+        obj = yaml.safe_load(objf)
+        assert obj['metadata']['namespace'] == 'myns'
+
+
+def test_postprocess_components_disabledref(tmp_path, capsys):
+    filter = _make_ns_filter('myns',
+                             enabled='${test_component:filter:enabled}')
+    testf, config, inventory, target, components = _setup(tmp_path, filter)
+    inventory['parameters']['test_component']['filter'] = {
+        'enabled': False,
+    }
+    postprocess_components(config, inventory, target, components)
+    assert testf.exists()
+    with open(testf) as objf:
+        obj = yaml.safe_load(objf)
+        assert obj['metadata']['namespace'] == 'untouched'
+    captured = capsys.readouterr()
+    assert "Skipping disabled filter" in captured.out