TIKA-4577 -- improve metadata filter serialization via ParseContext. (#2461)

tballison · web-flow · commit d09383aa491c · 2025-12-16T15:28:31.000-05:00
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/PipesClientTest.java b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/PipesClientTest.java
@@ -28,6 +28,7 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
+import org.apache.tika.config.ConfigContainer;
 import org.apache.tika.config.TikaTaskTimeout;
 import org.apache.tika.config.loader.TikaJsonConfig;
 import org.apache.tika.metadata.Metadata;
@@ -41,6 +42,7 @@
 import org.apache.tika.pipes.api.PipesResult;
 import org.apache.tika.pipes.api.emitter.EmitKey;
 import org.apache.tika.pipes.api.fetcher.FetchKey;
+import org.apache.tika.serialization.ParseContextUtils;
 
 public class PipesClientTest {
     String fetcherName = "fsf";
@@ -104,6 +106,74 @@ public void testMetadataListFilter(@TempDir Path tmp) throws Exception {
         assertEquals(4, Integer.parseInt(metadata.get("X-TIKA:attachment_count")));
     }
 
+    @Test
+    public void testMetadataFilterFromJsonConfig(@TempDir Path tmp) throws Exception {
+        // Test that metadata filters specified as JSON array in ConfigContainer
+        // are properly resolved and applied during pipe processing.
+        // This tests the full serialization/deserialization flow.
+        ParseContext parseContext = new ParseContext();
+        ConfigContainer configContainer = new ConfigContainer();
+        configContainer.set("metadata-filters", """
+            [
+              "mock-upper-case-filter"
+            ]
+        """);
+        parseContext.set(ConfigContainer.class, configContainer);
+
+        // Resolve the config to actual MetadataFilter instances
+        ParseContextUtils.resolveAll(parseContext, PipesClientTest.class.getClassLoader());
+
+        // Verify the filter was resolved
+        MetadataFilter resolvedFilter = parseContext.get(MetadataFilter.class);
+        Assertions.assertNotNull(resolvedFilter, "MetadataFilter should be resolved from ConfigContainer");
+        assertEquals(CompositeMetadataFilter.class, resolvedFilter.getClass());
+
+        PipesClient pipesClient = init(tmp, testDoc);
+        PipesResult pipesResult = pipesClient.process(
+                new FetchEmitTuple(testDoc, new FetchKey(fetcherName, testDoc),
+                        new EmitKey(), new Metadata(), parseContext, FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP));
+
+        Assertions.assertNotNull(pipesResult.emitData().getMetadataList());
+        assertEquals(1, pipesResult.emitData().getMetadataList().size());
+        Metadata metadata = pipesResult.emitData().getMetadataList().get(0);
+        // MockUpperCaseFilter uppercases all metadata values
+        assertEquals("TESTOVERLAPPINGTEXT.PDF", metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY));
+    }
+
+    @Test
+    public void testMultipleMetadataFiltersFromJsonConfig(@TempDir Path tmp) throws Exception {
+        // Test multiple filters specified as JSON array
+        ParseContext parseContext = new ParseContext();
+        ConfigContainer configContainer = new ConfigContainer();
+        configContainer.set("metadata-filters", """
+            [
+              "attachment-counting-list-filter",
+              "mock-upper-case-filter"
+            ]
+        """);
+        parseContext.set(ConfigContainer.class, configContainer);
+
+        // Resolve the config to actual MetadataFilter instances
+        ParseContextUtils.resolveAll(parseContext, PipesClientTest.class.getClassLoader());
+
+        String testFile = "mock-embedded.xml";
+        PipesClient pipesClient = init(tmp, testFile);
+
+        PipesResult pipesResult = pipesClient.process(
+                new FetchEmitTuple(testFile, new FetchKey(fetcherName, testFile),
+                        new EmitKey(), new Metadata(), parseContext, FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP));
+
+        Assertions.assertNotNull(pipesResult.emitData().getMetadataList());
+        assertEquals(5, pipesResult.emitData().getMetadataList().size());
+        Metadata metadata = pipesResult.emitData().getMetadataList().get(0);
+
+        // AttachmentCountingListFilter should have added the count
+        assertEquals(4, Integer.parseInt(metadata.get("X-TIKA:attachment_count")));
+
+        // MockUpperCaseFilter should have uppercased the resource name
+        assertEquals("MOCK-EMBEDDED.XML", metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY));
+    }
+
     @Test
     public void testTimeout(@TempDir Path tmp) throws Exception {
         //TODO -- figure out how to test pipes server timeout alone
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextDeserializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextDeserializer.java
@@ -200,6 +200,9 @@ public static ParseContext readParseContext(JsonNode jsonNode) throws IOExceptio
             parseContext.set(ConfigContainer.class, configContainer);
         }
 
+        // Resolve array configs (e.g., "metadata-filters") and non-SelfConfiguring components
+        ParseContextUtils.resolveAll(parseContext, ParseContextDeserializer.class.getClassLoader());
+
         return parseContext;
     }
 }
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextUtils.java b/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextUtils.java
@@ -19,17 +19,21 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.tika.config.ConfigContainer;
 import org.apache.tika.config.JsonConfig;
 import org.apache.tika.config.loader.ComponentInfo;
+import org.apache.tika.config.loader.ComponentInstantiator;
 import org.apache.tika.config.loader.ComponentRegistry;
 import org.apache.tika.config.loader.TikaObjectMapperFactory;
 import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.metadata.filter.CompositeMetadataFilter;
 import org.apache.tika.metadata.filter.MetadataFilter;
 import org.apache.tika.parser.ParseContext;
 
@@ -68,6 +72,20 @@ public class ParseContextUtils {
             // Add other known interfaces as needed
     );
 
+    /**
+     * Mapping of array config keys to their context keys and composite wrapper factories.
+     * Key: config name (e.g., "metadata-filters")
+     * Value: (contextKey, componentInterface)
+     */
+    private static final Map<String, ArrayConfigInfo> ARRAY_CONFIGS = Map.of(
+            "metadata-filters", new ArrayConfigInfo(MetadataFilter.class, MetadataFilter.class)
+    );
+
+    /**
+     * Holds information about how to process array configs.
+     */
+    private record ArrayConfigInfo(Class<?> contextKey, Class<?> componentInterface) {}
+
     /**
      * Resolves all friendly-named components from ConfigContainer and adds them to ParseContext.
      * <p>
@@ -102,12 +120,27 @@ public static void resolveAll(ParseContext context, ClassLoader classLoader) {
 
         List<String> resolvedKeys = new ArrayList<>();
 
+        // First, process known array configs (e.g., "metadata-filters")
+        // These don't depend on the other-configs registry
+        for (String friendlyName : new ArrayList<>(container.getKeys())) {
+            if (ARRAY_CONFIGS.containsKey(friendlyName)) {
+                JsonConfig jsonConfig = container.get(friendlyName, null);
+                if (jsonConfig != null && resolveArrayConfig(friendlyName, jsonConfig, context, classLoader)) {
+                    resolvedKeys.add(friendlyName);
+                }
+            }
+        }
+
+        // Then, try to load the "other-configs" registry for single component configs
         try {
-            // Load the "other-configs" registry which includes parse-context components
             ComponentRegistry registry = new ComponentRegistry("other-configs", classLoader);
 
-            // Iterate through all configs in the container
             for (String friendlyName : container.getKeys()) {
+                // Skip already resolved array configs
+                if (resolvedKeys.contains(friendlyName)) {
+                    continue;
+                }
+
                 JsonConfig jsonConfig = container.get(friendlyName, null);
                 if (jsonConfig == null) {
                     continue;
@@ -143,7 +176,8 @@ public static void resolveAll(ParseContext context, ClassLoader classLoader) {
                 }
             }
         } catch (TikaConfigException e) {
-            LOG.warn("Failed to load other-configs registry for parse-context resolution", e);
+            // other-configs registry not available - that's okay, array configs were still processed
+            LOG.debug("other-configs registry not available: {}", e.getMessage());
         }
 
         // Remove resolved configs from the container
@@ -191,4 +225,102 @@ private static Class<?> determineContextKey(ComponentInfo info, String friendlyN
         // Use the single matched interface, or fall back to the component class
         return matches.isEmpty() ? info.componentClass() : matches.get(0);
     }
+
+    /**
+     * Resolves an array config entry (e.g., "metadata-filters") to a composite component.
+     * <p>
+     * The array can contain either strings (friendly names) or objects:
+     * <pre>
+     * ["filter-name-1", "filter-name-2"]              // String shorthand
+     * [{"filter-name-1": {}}, {"filter-name-2": {}}]  // Object format
+     * </pre>
+     *
+     * @param configName the config name (e.g., "metadata-filters")
+     * @param jsonConfig the JSON configuration (should be an array)
+     * @param context the ParseContext to add the resolved component to
+     * @param classLoader the ClassLoader to use for loading component classes
+     * @return true if resolution was successful
+     */
+    @SuppressWarnings("unchecked")
+    private static boolean resolveArrayConfig(String configName, JsonConfig jsonConfig,
+                                              ParseContext context, ClassLoader classLoader) {
+        ArrayConfigInfo configInfo = ARRAY_CONFIGS.get(configName);
+        if (configInfo == null) {
+            return false;
+        }
+
+        try {
+            JsonNode arrayNode = MAPPER.readTree(jsonConfig.json());
+            if (!arrayNode.isArray()) {
+                LOG.warn("Expected array for '{}', got: {}", configName, arrayNode.getNodeType());
+                return false;
+            }
+
+            List<Object> components = new ArrayList<>();
+
+            for (JsonNode item : arrayNode) {
+                String typeName;
+                JsonNode configNode;
+
+                if (item.isTextual()) {
+                    // String shorthand: "component-name"
+                    typeName = item.asText();
+                    configNode = MAPPER.createObjectNode();
+                } else if (item.isObject() && item.size() == 1) {
+                    // Object format: {"component-name": {...}}
+                    typeName = item.fieldNames().next();
+                    configNode = item.get(typeName);
+                } else {
+                    LOG.warn("Unexpected item format in '{}': {}", configName, item);
+                    continue;
+                }
+
+                try {
+                    Object component = ComponentInstantiator.instantiate(
+                            typeName, configNode, MAPPER, classLoader);
+                    components.add(component);
+                    LOG.debug("Instantiated '{}' for '{}'", typeName, configName);
+                } catch (TikaConfigException e) {
+                    LOG.warn("Failed to instantiate '{}' for '{}': {}", typeName, configName, e.getMessage());
+                }
+            }
+
+            // Create the composite and add to ParseContext
+            if (!components.isEmpty()) {
+                Object composite = createComposite(configName, components, configInfo);
+                if (composite != null) {
+                    context.set((Class) configInfo.contextKey(), composite);
+                    LOG.debug("Resolved '{}' -> {} with {} components",
+                            configName, composite.getClass().getSimpleName(), components.size());
+                    return true;
+                }
+            }
+        } catch (IOException e) {
+            LOG.warn("Failed to parse array config '{}': {}", configName, e.getMessage());
+        }
+
+        return false;
+    }
+
+    /**
+     * Creates a composite component from a list of individual components.
+     *
+     * @param configName the config name (for error messages)
+     * @param components the list of components
+     * @param configInfo the array config info
+     * @return the composite component, or null if creation failed
+     */
+    @SuppressWarnings("unchecked")
+    private static Object createComposite(String configName, List<Object> components,
+                                          ArrayConfigInfo configInfo) {
+        // Handle known composite types
+        if (configInfo.componentInterface() == MetadataFilter.class) {
+            List<MetadataFilter> filters = (List<MetadataFilter>) (List<?>) components;
+            return new CompositeMetadataFilter(filters);
+        }
+
+        // Add more composite types as needed
+        LOG.warn("No composite factory for '{}'", configName);
+        return null;
+    }
 }
diff --git a/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java b/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java
@@ -35,6 +35,9 @@
 import org.apache.tika.config.loader.TikaObjectMapperFactory;
 import org.apache.tika.extractor.DocumentSelector;
 import org.apache.tika.extractor.SkipEmbeddedDocumentSelector;
+import org.apache.tika.metadata.filter.AttachmentCountingListFilter;
+import org.apache.tika.metadata.filter.CompositeMetadataFilter;
+import org.apache.tika.metadata.filter.MetadataFilter;
 import org.apache.tika.parser.ParseContext;
 
 /**
@@ -296,6 +299,29 @@ public void testProgrammaticObjectsWithoutFriendlyName() throws Exception {
         assertEquals(0, root.size(), "Objects without friendly names should not be serialized");
     }
 
+    @Test
+    public void testMetadataList() throws Exception {
+        ConfigContainer configContainer = new ConfigContainer();
+        configContainer.set("metadata-filters", """
+            [
+              "attachment-counting-list-filter",
+              "mock-upper-case-filter"
+            ]
+        """);
+        ParseContext parseContext = new ParseContext();
+        parseContext.set(ConfigContainer.class, configContainer);
+
+        ObjectMapper mapper = createMapper();
+        String json = mapper.writeValueAsString(parseContext);
+
+        ParseContext deser = mapper.readValue(json, ParseContext.class);
+        MetadataFilter resolvedFilter = deser.get(MetadataFilter.class);
+        assertNotNull(resolvedFilter, "MetadataFilter should be resolved");
+        assertEquals(CompositeMetadataFilter.class, resolvedFilter.getClass());
+        CompositeMetadataFilter deserFilter = (CompositeMetadataFilter) resolvedFilter;
+        assertEquals(AttachmentCountingListFilter.class, deserFilter.getFilters().get(0).getClass());
+    }
+
     @Test
     public void testContextKeyDeserialization() throws Exception {
         // Test that components with @TikaComponent(contextKey=...) are stored

Original file line number	Diff line number	Diff line change
`@@ -200,6 +200,9 @@ public static ParseContext readParseContext(JsonNode jsonNode) throws IOExceptio`
`200`	`200`	`parseContext.set(ConfigContainer.class, configContainer);`
`201`	`201`	`}`
`202`	`202`
	`203`	`+ // Resolve array configs (e.g., "metadata-filters") and non-SelfConfiguring components`
	`204`	`+ ParseContextUtils.resolveAll(parseContext, ParseContextDeserializer.class.getClassLoader());`
	`205`	`+`
`203`	`206`	`return parseContext;`
`204`	`207`	`}`
`205`	`208`	`}`