Feat(Block on missing config) (#42)

* Feat(Interface): Interface commit.

* Chore(): Implementation, half-working unit test.

* Chore(): Remove changes from charmcraft.yaml

* Chore(): Improve implementation.

* Chore(test): Fix unit tests.

* Chore(lint): Format code

* Chore(test): Fix unit test.

* Chore(test): Change implementation. Add a bunch of unit test.

* Chore(lint): Fix linting errors.

* Chore(lint): Fix linting errors.

* Chore(): Add non-optional to default example.

* Chore(test): Fix proxy test

* Chore(test): Add unit tests(WIP)

* Chore(test): Update unit tests.

* Chore(test): Update minor stuff.

* Chore(): Updated changelog.

* Chore(): Address comments.

* Chore(): Cleanup.

* Chore(): Cleanup.

* Chore(): Cleanup.

* Chore(lint): Make linter happy.

* Chore(lint): Fix newline.

* Chore(): Address comments.

* Chore(lint): Fix loc type.

* chore(test): Make it clear.

* chore(utils): Fix None prefix.

Author: alithethird

.github/workflows/integration_test.yaml

@@ -14,7 +14,7 @@ jobs:
       extra-arguments: -x --localstack-address 172.17.0.1
       pre-run-script: localstack-installation.sh
       charmcraft-channel: latest/edge
-      modules: '["test_charm.py", "test_cos.py", "test_database.py", "test_db_migration.py", "test_django.py", "test_django_integrations.py", "test_fastapi.py", "test_go.py", "test_integrations.py", "test_proxy.py", "test_workers.py", "test_tracing.py"]'
+      modules: '["test_charm.py", "test_cos.py", "test_database.py", "test_db_migration.py", "test_django.py", "test_django_integrations.py", "test_fastapi.py", "test_go.py", "test_integrations.py", "test_proxy.py", "test_workers.py", "test_tracing.py", "test_config.py"]'
       rockcraft-channel: latest/edge
       juju-channel: ${{ matrix.juju-version }}
       channel: 1.29-strict/stable

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## v1.3.0 - 2025-02-24
+
+### Changed
+
+* Added support for non-optional configuration options.
+
 ## v1.2.3 - 2025-02-07
 
 ### Changed

examples/fastapi/charm/charmcraft.yaml

@@ -58,6 +58,10 @@ config:
     user-defined-config:
       type: string
       description: Example of a user defined configuration.
+    non-optional-string:
+      description: Example of a non-optional string configuration option.
+      type: string
+      optional: False
 containers:
   app:
     resource: app-image

pyproject.toml

@@ -2,7 +2,7 @@
 # See LICENSE file for licensing details.
 [project]
 name = "paas-charm"
-version = "1.2.3"
+version = "1.3.0"
 description = "Companion library for 12-factor app support in Charmcraft & Rockcraft."
 readme = "README.md"
 authors = [

src/paas_charm/app.py

@@ -140,7 +140,7 @@ def gen_environment(self) -> dict[str, str]:
         """
         prefix = self.configuration_prefix
         env = {}
-        for app_config_key, app_config_value in self._charm_state.app_config.items():
+        for app_config_key, app_config_value in self._charm_state.user_defined_config.items():
             if isinstance(app_config_value, collections.abc.Mapping):
                 for k, v in app_config_value.items():
                     env[f"{prefix}{app_config_key.upper()}_{k.replace('-', '_').upper()}"] = (

src/paas_charm/charm.py

@@ -283,11 +283,13 @@ def get_framework_config(self) -> BaseModel:
                 for k, v in charm_config.items()
             },
         )
+
         try:
             return framework_config_class.model_validate(config)
         except ValidationError as exc:
-            error_message = build_validation_error_message(exc, underscore_to_dash=False)
-            raise CharmConfigInvalidError(f"invalid configuration: {error_message}") from exc
+            error_messages = build_validation_error_message(exc)
+            logger.error(error_messages.long)
+            raise CharmConfigInvalidError(error_messages.short) from exc
 
     @property
     def _container(self) -> Container:

src/paas_charm/charm_state.py

@@ -4,20 +4,29 @@
 """This module defines the CharmState class which represents the state of the charm."""
 import logging
 import os
+import pathlib
 import re
 import typing
 from dataclasses import dataclass, field
 from typing import Optional, Type, TypeVar
 
 from charms.data_platform_libs.v0.data_interfaces import DatabaseRequires
 from charms.redis_k8s.v0.redis import RedisRequires
-from pydantic import BaseModel, Extra, Field, ValidationError, ValidationInfo, field_validator
+from pydantic import (
+    BaseModel,
+    Extra,
+    Field,
+    ValidationError,
+    ValidationInfo,
+    create_model,
+    field_validator,
+)
 
 from paas_charm.databases import get_uri
 from paas_charm.exceptions import CharmConfigInvalidError
 from paas_charm.rabbitmq import RabbitMQRequires
 from paas_charm.secret_storage import KeySecretStorage
-from paas_charm.utils import build_validation_error_message
+from paas_charm.utils import build_validation_error_message, config_metadata
 
 logger = logging.getLogger(__name__)
 
@@ -55,7 +64,7 @@ class CharmState:  # pylint: disable=too-many-instance-attributes
 
     Attrs:
         framework_config: the value of the framework specific charm configuration.
-        app_config: user-defined configurations for the application.
+        user_defined_config: user-defined configurations for the application.
         secret_key: the charm managed application secret key.
         is_secret_storage_ready: whether the secret storage system is ready.
         proxy: proxy information.
@@ -66,7 +75,7 @@ def __init__(  # pylint: disable=too-many-arguments
         *,
         framework: str,
         is_secret_storage_ready: bool,
-        app_config: dict[str, int | str | bool | dict[str, str]] | None = None,
+        user_defined_config: dict[str, int | str | bool | dict[str, str]] | None = None,
         framework_config: dict[str, int | str] | None = None,
         secret_key: str | None = None,
         integrations: "IntegrationsState | None" = None,
@@ -77,15 +86,15 @@ def __init__(  # pylint: disable=too-many-arguments
         Args:
             framework: the framework name.
             is_secret_storage_ready: whether the secret storage system is ready.
-            app_config: User-defined configuration values for the application configuration.
+            user_defined_config: User-defined configuration values for the application.
             framework_config: The value of the framework application specific charm configuration.
             secret_key: The secret storage manager associated with the charm.
             integrations: Information about the integrations.
             base_url: Base URL for the service.
         """
         self.framework = framework
         self._framework_config = framework_config if framework_config is not None else {}
-        self._app_config = app_config if app_config is not None else {}
+        self._user_defined_config = user_defined_config if user_defined_config is not None else {}
         self._is_secret_storage_ready = is_secret_storage_ready
         self._secret_key = secret_key
         self.integrations = integrations or IntegrationsState()
@@ -116,16 +125,27 @@ def from_charm(  # pylint: disable=too-many-arguments
 
         Return:
             The CharmState instance created by the provided charm.
+
+        Raises:
+            CharmConfigInvalidError: If some parameter in invalid.
         """
-        app_config = {
+        user_defined_config = {
             k.replace("-", "_"): v
             for k, v in config.items()
-            if not any(k.startswith(prefix) for prefix in (f"{framework}-", "webserver-", "app-"))
+            if is_user_defined_config(k, framework)
         }
-        app_config = {
-            k: v for k, v in app_config.items() if k not in framework_config.dict().keys()
+        user_defined_config = {
+            k: v for k, v in user_defined_config.items() if k not in framework_config.dict().keys()
         }
 
+        app_config_class = app_config_class_factory(framework)
+        try:
+            app_config_class(**user_defined_config)
+        except ValidationError as exc:
+            error_messages = build_validation_error_message(exc, underscore_to_dash=True)
+            logger.error(error_messages.long)
+            raise CharmConfigInvalidError(error_messages.short) from exc
+
         saml_relation_data = None
         if integration_requirers.saml and (
             saml_data := integration_requirers.saml.get_relation_data()
@@ -153,7 +173,9 @@ def from_charm(  # pylint: disable=too-many-arguments
         return cls(
             framework=framework,
             framework_config=framework_config.dict(exclude_none=True),
-            app_config=typing.cast(dict[str, str | int | bool | dict[str, str]], app_config),
+            user_defined_config=typing.cast(
+                dict[str, str | int | bool | dict[str, str]], user_defined_config
+            ),
             secret_key=(
                 secret_storage.get_secret_key() if secret_storage.is_initialized else None
             ),
@@ -188,13 +210,13 @@ def framework_config(self) -> dict[str, str | int | bool]:
         return self._framework_config
 
     @property
-    def app_config(self) -> dict[str, str | int | bool | dict[str, str]]:
+    def user_defined_config(self) -> dict[str, str | int | bool | dict[str, str]]:
         """Get the value of user-defined application configurations.
 
         Returns:
             The value of user-defined application configurations.
         """
-        return self._app_config
+        return self._user_defined_config
 
     @property
     def secret_key(self) -> str:
@@ -351,9 +373,10 @@ def generate_relation_parameters(
     try:
         return parameter_type.parse_obj(relation_data)
     except ValidationError as exc:
-        error_message = build_validation_error_message(exc)
+        error_messages = build_validation_error_message(exc)
+        logger.error(error_messages.long)
         raise CharmConfigInvalidError(
-            f"Invalid {parameter_type.__name__} configuration: {error_message}"
+            f"Invalid {parameter_type.__name__}: {error_messages.short}"
         ) from exc
 
 
@@ -458,3 +481,76 @@ def validate_signing_certificate_exists(cls, certs: str, _: ValidationInfo) -> s
         if not certificate:
             raise ValueError("Missing x509certs. There should be at least one certificate.")
         return certificate
+
+
+def _create_config_attribute(option_name: str, option: dict) -> tuple[str, tuple]:
+    """Create the configuration attribute.
+
+    Args:
+        option_name: Name of the configuration option.
+        option: The configuration option data.
+
+    Raises:
+        ValueError: raised when the option type is not valid.
+
+    Returns:
+        A tuple constructed from attribute name and type.
+    """
+    option_name = option_name.replace("-", "_")
+    optional = option.get("optional") is not False
+    config_type_str = option.get("type")
+
+    config_type: type[bool] | type[int] | type[float] | type[str] | type[dict]
+    match config_type_str:
+        case "boolean":
+            config_type = bool
+        case "int":
+            config_type = int
+        case "float":
+            config_type = float
+        case "string":
+            config_type = str
+        case "secret":
+            config_type = dict
+        case _:
+            raise ValueError(f"Invalid option type: {config_type_str}.")
+
+    type_tuple: tuple = (config_type, Field())
+    if optional:
+        type_tuple = (config_type | None, None)
+
+    return (option_name, type_tuple)
+
+
+def app_config_class_factory(framework: str) -> type[BaseModel]:
+    """App config class factory.
+
+    Args:
+        framework: The framework name.
+
+    Returns:
+        Constructed app config class.
+    """
+    config_options = config_metadata(pathlib.Path(os.getcwd()))["options"]
+    model_attributes = dict(
+        _create_config_attribute(option_name, config_options[option_name])
+        for option_name in config_options
+        if is_user_defined_config(option_name, framework)
+    )
+    # mypy doesn't like the model_attributes dict
+    return create_model("AppConfig", **model_attributes)  # type: ignore[call-overload]
+
+
+def is_user_defined_config(option_name: str, framework: str) -> bool:
+    """Check if a config option is user defined.
+
+    Args:
+        option_name: Name of the config option.
+        framework: The framework name.
+
+    Returns:
+        True if user defined config options, false otherwise.
+    """
+    return not any(
+        option_name.startswith(prefix) for prefix in (f"{framework}-", "webserver-", "app-")
+    )

src/paas_charm/utils.py

@@ -13,29 +13,67 @@
 from pydantic import ValidationError
 
 
+class ValidationErrorMessage(typing.NamedTuple):
+    """Class carrying status message and error log for pydantic validation errors.
+
+    Attrs:
+        short: Short error message to show in status message.
+        long: Detailed error message for logging.
+    """
+
+    short: str
+    long: str
+
+
 def build_validation_error_message(
     exc: ValidationError, prefix: str | None = None, underscore_to_dash: bool = False
-) -> str:
-    """Build a str with a list of error fields from a pydantic exception.
+) -> ValidationErrorMessage:
+    """Build a ValidationErrorMessage for error logging.
 
     Args:
         exc: ValidationError exception instance.
         prefix: Prefix to append to the error field names.
         underscore_to_dash: Replace underscores to dashes in the error field names.
 
     Returns:
-        The curated list of error fields ready to be used in an error message.
+        The ValidationErrorMessage for error logging..
     """
-    error_fields_unique = set(
-        itertools.chain.from_iterable(error["loc"] for error in exc.errors())
+    fields = set(
+        (
+            (
+                f'{prefix if prefix else ""}{".".join(str(loc) for loc in error["loc"])}'
+                if error["loc"]
+                else ""
+            ),
+            error["msg"],
+        )
+        for error in exc.errors()
     )
-    error_fields = (str(error_field) for error_field in error_fields_unique)
-    if prefix:
-        error_fields = (f"{prefix}{error_field}" for error_field in error_fields)
+
     if underscore_to_dash:
-        error_fields = (error_field.replace("_", "-") for error_field in error_fields)
-    error_field_str = " ".join(error_fields)
-    return error_field_str
+        fields = {(key.replace("_", "-"), value) for key, value in fields}
+
+    missing_fields = {}
+    invalid_fields = {}
+
+    for loc, msg in fields:
+        if "required" in msg.lower():
+            missing_fields[loc] = msg
+        else:
+            invalid_fields[loc] = msg
+
+    short_str_missing = f"missing options: {', '.join(missing_fields)}" if missing_fields else ""
+    short_str_invalid = f"invalid options: {', '.join(invalid_fields)}" if invalid_fields else ""
+    short_str = f"{short_str_missing}\
+        {', ' if missing_fields and invalid_fields else ''}{short_str_invalid}"
+
+    long_str_lines = "\n".join(
+        f"- {key}: {value}"
+        for key, value in itertools.chain(missing_fields.items(), invalid_fields.items())
+    )
+    long_str = f"invalid configuration:\n{long_str_lines}"
+
+    return ValidationErrorMessage(short=short_str, long=long_str)
 
 
 def enable_pebble_log_forwarding() -> bool:
@@ -58,7 +96,7 @@ def enable_pebble_log_forwarding() -> bool:
 
 
 @functools.lru_cache
-def _config_metadata(charm_dir: pathlib.Path) -> dict:
+def config_metadata(charm_dir: pathlib.Path) -> dict:
     """Get charm configuration metadata for the given charm directory.
 
     Args:
@@ -95,7 +133,7 @@ def config_get_with_secret(
     Returns:
         The configuration value.
     """
-    metadata = _config_metadata(pathlib.Path(os.getcwd()))
+    metadata = config_metadata(pathlib.Path(os.getcwd()))
     config_type = metadata["options"][key]["type"]
     if config_type != "secret":
         return charm.config.get(key)