Reworked firewall definition

- renamed fields (breaking change) so they are clear
- added extra fields so they add all requirements
- added new yaml format checkers to check cidrs

assemble

@@ -22,6 +22,7 @@ install_package jsonschema jsonschema
 install_package json_schema_for_humans json_schema_for_humans
 install_package shellescape shellescape
 install_package colorama colorama
+install_package passlib passlib
 
 rm -rf build/src
 cp -a src build/src
@@ -31,6 +32,8 @@ cp -a build/lib*/python*/site-packages/jinja2 build/src/
 cp -a build/lib*/python*/site-packages/markupsafe build/src/
 # colorama for colorized output
 cp -a build/lib*/python*/site-packages/colorama build/src/
+# passlib provides pure-python password hashing for the jinjaextras plugin
+cp -a build/lib*/python*/site-packages/passlib build/src/
 
 
 find build/src -type d -name "__pycache__" -exec rm -rf {} \; 2>/dev/null | true 

src/__main__.py

@@ -126,11 +126,16 @@ def write_state(config, plugin_api):
         exports = modifier(exports)
     exports = utils.render_strings_recursively(exports, config["templatevars"])
     for key in exports:
-        err, _ = yamllib.read_yaml_string(json.dumps(exports[key]), f"exports {key} in appdefinition",
+        # Round-trip through the import schema so consumers see defaults
+        # filled in (e.g. `firewall.open[*].ips` defaulting to `[]`),
+        # not just the raw author input.  Without this, importers have
+        # to defensively `| default([])` every field that has a default.
+        err, validated = yamllib.read_yaml_string(json.dumps(exports[key]), f"exports {key} in appdefinition",
                 utils.get_resource_or_default(f"import_{key}.yml","importother.yml"),  f"Built-in import definition for {key}")
         if err!="":
             print(f"Fatal: the export definition is not valid: {err}", file=sys.stderr)
             sys.exit(1)
+        exports[key] = validated
     state = {
         "ports": config["ports"],
         "exports": exports,
@@ -350,6 +355,29 @@ def main():
     if err!="":
         print(err, file=sys.stderr)
         sys.exit(1)
+
+    os.chdir(os.path.expanduser("~"))
+    plugin_api = utils.PluginAPI()
+    plugin_api.appinfo = config['appinfo']
+    utils.loadplugins(plugin_api, config["ppmconfig"].get("plugins", []))
+    utils.loadplugins(plugin_api, config["appconfig"].get("plugins", []))
+    utils.loadplugins(plugin_api, config["appinfo"].get("plugins", []))
+    utils.autoload_plugins(plugin_api)
+
+    # Apply appinfo patchers (e.g. the patchappinfo plugin) before port
+    # resolution and before templatevars is built, so any ports/templatefiles/
+    # exports they add flow through the normal pipeline.
+    if plugin_api.appinfo_patchers:
+        minimal_templatevars = {
+            "name": config['appconfig']['name'],
+            "tags": config["appconfig"]["tags"],
+            "config": config["appconfig"]["config"],
+            "const": config['appinfo']['const'],
+        }
+        for patcher in plugin_api.appinfo_patchers:
+            config["appinfo"] = patcher(config["appinfo"], minimal_templatevars)
+        plugin_api.appinfo = config['appinfo']
+
     config["ports"] = resolve_ports(config)
 
     # Some variables need some work, ie to expand them in a way so they are useable: do this now:
@@ -369,14 +397,6 @@ def main():
         "const": config['appinfo']['const'],
     }
 
-    os.chdir(os.path.expanduser("~"))
-    plugin_api = utils.PluginAPI()
-    plugin_api.appinfo = config['appinfo']
-    utils.loadplugins(plugin_api, config["ppmconfig"].get("plugins", []))
-    utils.loadplugins(plugin_api, config["appconfig"].get("plugins", []))
-    utils.loadplugins(plugin_api, config["appinfo"].get("plugins", []))
-    utils.autoload_plugins(plugin_api)
-
     if not plugin_api._backups_handled:
         print(
             "Fatal: no plugin called backups_handled().  Every appinfo must load a backup plugin "
@@ -416,11 +436,14 @@ def main():
         sys.exit(0)
 
     if args.command == "setup":
-        neededchange = False
-        neededchange = neededchange or restore_backup(config, plugin_api)
-        neededchange = neededchange or setup_files(config, args, plugin_api)
-        if args.start:
-            neededchange = neededchange or utils.ensure_started(maintarget)
+        # Each step must run unconditionally — `neededchange or step()`
+        # short-circuits and skips the step once anything earlier already
+        # returned True.  We track whether anything changed by ORing the
+        # results after the fact.
+        restored = restore_backup(config, plugin_api)
+        files_changed = setup_files(config, args, plugin_api)
+        started = utils.ensure_started(maintarget) if args.start else False
+        neededchange = restored or files_changed or started
         if not neededchange:
             print("No changes have been made, everything was already ok")
 

src/builtin_plugins/jinjaextras.py

@@ -0,0 +1,69 @@
+"""Grab-bag of small jinja helpers that are useful across appinfos.
+
+Add new filters here rather than spawning a one-filter plugin per helper.
+This plugin is autoloaded; it has no config and no side effects beyond
+registering filters."""
+
+import hashlib
+
+
+PLUGIN_CONFIG_SCHEMA = """
+type: object
+additionalProperties: false
+"""
+
+
+_SUPPORTED_ALGORITHMS = ("bcrypt", "apr1", "sha256_crypt", "sha512_crypt")
+
+
+def _stable_salt(password, length, charset):
+    """Derive a deterministic salt of `length` characters drawn from `charset`
+    by hashing the password.  The same password always yields the same salt,
+    which is the whole point: it lets us render an htpasswd-style file from a
+    template idempotently."""
+    digest = hashlib.sha256(password.encode("utf-8")).digest()
+    return "".join(charset[b % len(charset)] for b in digest[:length])
+
+
+def stable_password_hash(password, algorithm="bcrypt"):
+    """Hash `password` with `algorithm`, using a salt deterministically derived
+    from the password itself.  Output is a crypt(3)-style string suitable for
+    htpasswd files and similar.
+
+    The fixed-salt design makes the hash reproducible across runs, so a
+    rendered file does not churn on every `ppm setup`.  This is only safe when
+    the plaintext password is already stored next to the hash (e.g. in the
+    ppm config on the same host); do NOT use this filter for passwords that
+    would otherwise live only in hashed form."""
+    if algorithm not in _SUPPORTED_ALGORITHMS:
+        raise ValueError(
+            f"stable_password_hash: unsupported algorithm '{algorithm}'. "
+            f"Supported: {', '.join(_SUPPORTED_ALGORITHMS)}"
+        )
+
+    if algorithm == "bcrypt":
+        from passlib.hash import bcrypt
+        # bcrypt salts are 128 bits encoded as 22 bcrypt-base64 chars.  The last
+        # char only carries 2 meaningful bits, so it must be one of ".Oeu" or
+        # passlib will warn that the padding bits are non-zero.
+        salt = _stable_salt(password, 21, bcrypt.salt_chars) + "."
+        return bcrypt.using(salt=salt, rounds=12, ident="2y").hash(password)
+
+    if algorithm == "apr1":
+        from passlib.hash import apr_md5_crypt
+        salt = _stable_salt(password, 8, apr_md5_crypt.salt_chars)
+        return apr_md5_crypt.using(salt=salt).hash(password)
+
+    if algorithm == "sha256_crypt":
+        from passlib.hash import sha256_crypt
+        salt = _stable_salt(password, 16, sha256_crypt.salt_chars)
+        return sha256_crypt.using(salt=salt, rounds=535000).hash(password)
+
+    if algorithm == "sha512_crypt":
+        from passlib.hash import sha512_crypt
+        salt = _stable_salt(password, 16, sha512_crypt.salt_chars)
+        return sha512_crypt.using(salt=salt, rounds=656000).hash(password)
+
+
+def register(api, plugin_config):
+    api.register_jinjafilter("stable_password_hash", stable_password_hash)

src/builtin_plugins/patchappinfo.py

@@ -0,0 +1,191 @@
+import copy
+import json
+import sys
+
+import utils
+import yamllib
+
+
+PLUGIN_CONFIG_SCHEMA = """
+type: object
+additionalProperties: false
+properties:
+    patches:
+        type: array
+        default: []
+        description: |
+            Ordered list of conditional patches.  Each entry has an `if`
+            jinja expression evaluated against the minimal templatevars
+            (config, const, tags, name).  When it renders to "true" the
+            corresponding JSON Patch op lists are applied to the
+            appinfo's ports, templatefiles and exports scopes.  Earlier
+            patches' effects are visible to later patches' ops, but the
+            `if` expression only sees config (which never changes), so
+            patch order matters only for ops, not for conditions.
+        items:
+            type: object
+            additionalProperties: false
+            properties:
+                if:
+                    type: string
+                    description: Jinja expression.  Must render to exactly "true" or "false" after stripping.
+                ports:
+                    type: array
+                    default: []
+                    items:
+                        "$ref": "#/definitions/op"
+                templatefiles:
+                    type: array
+                    default: []
+                    items:
+                        "$ref": "#/definitions/op"
+                exports:
+                    type: array
+                    default: []
+                    items:
+                        "$ref": "#/definitions/op"
+            required:
+                - if
+            anyOf:
+                - required: [ports]
+                - required: [templatefiles]
+                - required: [exports]
+required:
+    - patches
+definitions:
+    op:
+        type: object
+        properties:
+            op:
+                type: string
+                enum: [add, remove, replace]
+            path:
+                type: string
+                description: JSON Pointer (RFC 6901).  Use /- to append to an array.
+            value:
+                description: Required for add and replace.
+        required:
+            - op
+            - path
+"""
+
+
+_PATCHABLE_SCOPES = ("ports", "templatefiles", "exports")
+
+
+def _unescape_token(token):
+    return token.replace("~1", "/").replace("~0", "~")
+
+
+def _resolve_parent(doc, path):
+    """Walk all but the last path segment.  Returns (parent, last_token)."""
+    if path == "":
+        raise ValueError("empty path is not valid for this patcher (cannot replace whole scope)")
+    if not path.startswith("/"):
+        raise ValueError(f"path must start with /: {path!r}")
+    tokens = [_unescape_token(t) for t in path.split("/")[1:]]
+    parent = doc
+    for token in tokens[:-1]:
+        if isinstance(parent, list):
+            parent = parent[int(token)]
+        elif isinstance(parent, dict):
+            parent = parent[token]
+        else:
+            raise ValueError(f"cannot descend into {type(parent).__name__} at segment {token!r}")
+    return parent, tokens[-1]
+
+
+def _apply_op(doc, op):
+    parent, last = _resolve_parent(doc, op["path"])
+    kind = op["op"]
+    if kind == "add":
+        if "value" not in op:
+            raise ValueError("'add' op requires 'value'")
+        value = op["value"]
+        if isinstance(parent, list):
+            if last == "-":
+                parent.append(value)
+            else:
+                parent.insert(int(last), value)
+        elif isinstance(parent, dict):
+            parent[last] = value
+        else:
+            raise ValueError(f"cannot add into {type(parent).__name__}")
+    elif kind == "remove":
+        if isinstance(parent, list):
+            del parent[int(last)]
+        elif isinstance(parent, dict):
+            del parent[last]
+        else:
+            raise ValueError(f"cannot remove from {type(parent).__name__}")
+    elif kind == "replace":
+        if "value" not in op:
+            raise ValueError("'replace' op requires 'value'")
+        value = op["value"]
+        if isinstance(parent, list):
+            parent[int(last)] = value
+        elif isinstance(parent, dict):
+            if last not in parent:
+                raise ValueError(f"cannot replace missing key {last!r}")
+            parent[last] = value
+        else:
+            raise ValueError(f"cannot replace into {type(parent).__name__}")
+    else:
+        raise ValueError(f"unknown op {kind!r}")
+
+
+def _eval_condition(expr, templatevars, index):
+    try:
+        rendered = utils.render_template(expr, templatevars).strip()
+    except Exception as e:
+        print(f"patchappinfo: failed to render `if` of patch #{index}: {e}", file=sys.stderr)
+        sys.exit(1)
+    if rendered == "true":
+        return True
+    if rendered == "false":
+        return False
+    print(
+        f"patchappinfo: `if` of patch #{index} rendered to {rendered!r}, expected exactly 'true' or 'false'",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+
+def register(api, plugin_config):
+    patches = plugin_config["patches"]
+    if not patches:
+        return
+
+    appinfo_schema = utils.get_resource("appinfo.yml")
+
+    def patcher(appinfo, minimal_templatevars):
+        appinfo = copy.deepcopy(appinfo)
+        for index, patch in enumerate(patches):
+            if not _eval_condition(patch["if"], minimal_templatevars, index):
+                continue
+            for scope in _PATCHABLE_SCOPES:
+                for op in patch.get(scope, []):
+                    try:
+                        _apply_op(appinfo[scope], op)
+                    except Exception as e:
+                        print(
+                            f"patchappinfo: patch #{index} op on {scope} failed: {e}",
+                            file=sys.stderr,
+                        )
+                        sys.exit(1)
+            # Re-validate (and re-default) the full appinfo after each patch
+            # entry so that omitted defaults get filled in and any schema
+            # violations are reported against the patch that introduced them.
+            err, revalidated = yamllib.read_yaml_string(
+                json.dumps(appinfo),
+                f"appinfo after patchappinfo patch #{index}",
+                appinfo_schema,
+                "RESOURCE/appinfo.yml",
+            )
+            if err != "":
+                print(err, file=sys.stderr)
+                sys.exit(1)
+            appinfo = revalidated
+        return appinfo
+
+    api.register_appinfo_patcher(patcher)

src/resources/import_firewall.yml

@@ -5,21 +5,22 @@ type: object
 additionalProperties: false
 properties:
   redirect:
-    description: An array of ports that we need to redirect to a local port. The ports also will be opened up.
+    description: An array of ports the firewall should redirect to a local port.  The redirected-to port also gets opened up automatically.
     type: array
     default: []
     items:
       type: object
       additionalProperties: false
       properties:
-        from:
-          # These should be an integer, but that would make it impossible for us to use templates, so we force strings for now
+        listenport:
+          # Ports should be integers, but jinja templates render to strings;
+          # we accept the string form so appinfos can use "{{ ports.foo }}".
           type: string
-          description: The port that the packets will arrive at
+          description: The port the firewall listens on (the public-facing port packets arrive at).
           pattern: "^[0-9]{1,5}$"
-        to:
+        redirecttoport:
           type: string
-          description: The ports the packet should be redirected to
+          description: The local port the firewall redirects matching packets to.
           pattern: "^[0-9]{1,5}$"
         proto:
           type: string
@@ -35,16 +36,34 @@ properties:
             - ipv4
             - ipv6
           default: ipv4
-        ip:
-          type: string
-          description: The ip address
-          # We currently do not properly validate ip's, as you can give a v4 addres when version is set to v6, but this should cover most accidental mistakes
-          format:
-            - ipv4
-            - ipv6
+        listenips:
+          type: array
+          default: []
+          description: The host-side IPs to match the redirect on (iptables `-d <ip>`).  If empty or omitted, the rule applies on all interfaces.  Do not list wildcard addresses (0.0.0.0 / ::) here; leave the array empty instead.
+          items:
+            type: string
+        allowedfromips:
+          type: array
+          default: []
+          description: Source IPs/CIDR ranges allowed to use this redirect (iptables `-s <cidr>`).  If empty or omitted, packets from any source are accepted.  Accepts plain IPs as well as CIDR ranges (e.g. "10.0.0.0/8").
+          items:
+            type: string
       required:
-        - to
-        - from
+        - listenport
+        - redirecttoport
+      # Require every listenip / allowedfromips entry to match the declared version (default ipv4).
+      allOf:
+        - if:
+            properties: { version: { const: ipv6 } }
+            required: [version]
+          then:
+            properties:
+              listenips: { items: { format: ipv6 } }
+              allowedfromips: { items: { format: ipv6-cidr } }
+          else:
+            properties:
+              listenips: { items: { format: ipv4 } }
+              allowedfromips: { items: { format: ipv4-cidr } }
   open:
     description: A list of all ports that should be open in the firewall
     type: array
@@ -53,9 +72,9 @@ properties:
       type: object
       additionalProperties: false
       properties:
-        port:
+        listenport:
           type: string
-          description: The port that the packets will arrive at
+          description: The port the firewall listens on.
           pattern: "^[0-9]{1,5}$"
         proto:
           type: string
@@ -71,6 +90,30 @@ properties:
             - ipv4
             - ipv6
           default: ipv4
+        listenips:
+          type: array
+          default: []
+          description: The host-side IPs to open the port on (iptables `-d <ip>`).  If empty or omitted, the port is opened on all interfaces.  Otherwise the port is opened only on the listed addresses (one iptables rule per entry).  Do not list wildcard addresses (0.0.0.0 / ::) here; leave the array empty instead.
+          items:
+            type: string
+        allowedfromips:
+          type: array
+          default: []
+          description: Source IPs/CIDR ranges allowed to reach this port (iptables `-s <cidr>`).  If empty or omitted, packets from any source are accepted.  Accepts plain IPs as well as CIDR ranges (e.g. "10.0.0.0/8").
+          items:
+            type: string
       required:
-        - port
-        - from
+        - listenport
+      # Require every listenip / allowedfromips entry to match the declared version (default ipv4).
+      allOf:
+        - if:
+            properties: { version: { const: ipv6 } }
+            required: [version]
+          then:
+            properties:
+              listenips: { items: { format: ipv6 } }
+              allowedfromips: { items: { format: ipv6-cidr } }
+          else:
+            properties:
+              listenips: { items: { format: ipv4 } }
+              allowedfromips: { items: { format: ipv4-cidr } }

src/utils.py

@@ -4,6 +4,8 @@ import subprocess
 import sys
 import os
 
+import yamllib
+
 ###### Template
 
 import jinja2
@@ -33,6 +35,17 @@ def render_template(template, variables):
 
 
 def render_strings_recursively(data, variables):
+    # `!yaml`-tagged scalars (see yamllib) render through jinja, then their
+    # output is parsed as YAML and substituted in place.  This lets an
+    # appinfo produce a variable-length list/dict from config, which plain
+    # leaf-string rendering can't express.
+    if isinstance(data, yamllib.YamlRenderString):
+        rendered = render_template(str(data), variables)
+        try:
+            parsed = yamllib.yaml.safe_load(rendered)
+        except yamllib.yaml.YAMLError as e:
+            raise ValueError(f"!yaml tagged value rendered to invalid YAML: {e}\nrendered value was: {rendered!r}")
+        return render_strings_recursively(parsed, variables)
     if isinstance(data, str):
         return render_template(data, variables)
     elif isinstance(data, list):
@@ -150,6 +163,7 @@ def ensure_started(service):
 
 AUTOLOAD_PLUGINS = [
     {"type": "builtin", "name": "restic"},
+    {"type": "builtin", "name": "jinjaextras"},
 ]
 
 
@@ -162,6 +176,7 @@ class PluginAPI:
         self.extra_template_files = []
         self.extra_enableunits = []
         self.export_modifiers = []
+        self.appinfo_patchers = []
         self.skip_restore = False
         self.skip_take = False
         self.skip_take_reason = None
@@ -208,6 +223,15 @@ class PluginAPI:
         overwrite keys)."""
         self.export_modifiers.append(func)
 
+    def register_appinfo_patcher(self, func):
+        """Register a plugin hook that runs *before* port resolution and the
+        templatevars dict is built, so it can mutate the appinfo (ports,
+        templatefiles, exports) based on the user's config.  The callable
+        receives (appinfo, minimal_templatevars) and must return the new
+        appinfo dict.  minimal_templatevars contains config, const, tags, name
+        only (ports and otherapps are not yet available)."""
+        self.appinfo_patchers.append(func)
+
 
 def _load_plugin_module(plugin):
     ptype = plugin["type"]

src/yamllib.py

@@ -1,6 +1,51 @@
+import ipaddress
 import yaml
 import jsonschema
 
+
+# Accept both plain IPs and CIDR ranges for the same version. `strict=False`
+# allows host bits to be set (e.g. "10.0.0.5/8"), which iptables accepts.
+# Pin to Draft 7's format set so we don't silently start asserting formats
+# (uuid, duration, ...) that Draft 7 schemas treat as unknown.
+_ppm_format_checker = jsonschema.FormatChecker(
+    formats=list(jsonschema.Draft7Validator.FORMAT_CHECKER.checkers),
+)
+
+
+@_ppm_format_checker.checks("ipv4-cidr", raises=ValueError)
+def _check_ipv4_cidr(value):
+    if not isinstance(value, str):
+        return True
+    ipaddress.IPv4Network(value, strict=False)
+    return True
+
+
+@_ppm_format_checker.checks("ipv6-cidr", raises=ValueError)
+def _check_ipv6_cidr(value):
+    if not isinstance(value, str):
+        return True
+    ipaddress.IPv6Network(value, strict=False)
+    return True
+
+
+class YamlRenderString(str):
+    """Marker for scalars tagged `!yaml` in source YAML.
+
+    Such a scalar is preserved as a string through schema-free load steps
+    (e.g. exports in an appinfo).  Later, `render_strings_recursively`
+    detects the marker, renders the string through jinja, and parses the
+    output as YAML, substituting the resulting structure in place.  This
+    lets a template produce a variable-length list/dict, which plain
+    leaf-string rendering can't express.
+    """
+
+
+def _construct_yaml_render(loader, node):
+    return YamlRenderString(loader.construct_scalar(node))
+
+
+yaml.SafeLoader.add_constructor('!yaml', _construct_yaml_render)
+
 # This function loads a yaml string
 # you need to give it 2 parameters:
 # - the name of the file
@@ -39,7 +84,7 @@ def read_yaml_string(data, filename, schema, schemafilename, rootelement = None)
 
     myvalidator = extend_with_default(jsonschema.Draft7Validator)
     try:
-        myvalidator(schema).validate(data)
+        myvalidator(schema, format_checker=_ppm_format_checker).validate(data)
         return "", data
     except jsonschema.ValidationError as e:
         msg = f"The yaml file {filename} is invalid.  The following error occured: {e.message}\nPlace where the error is located: "