Adding use of ItemAdapter to prevent assumptions of item nature

#353

Author: further-reading

spidermon/contrib/scrapy/extensions.py

@@ -2,6 +2,7 @@
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from twisted.internet.task import LoopingCall
+from itemadapter import ItemAdapter
 
 from spidermon import MonitorSuite
 from spidermon.contrib.scrapy.runners import SpiderMonitorRunner
@@ -137,7 +138,8 @@ def _count_item(self, item, skip_none_values, item_count_stat=None):
             item_count_stat = f"spidermon_item_scraped_count/{item_type}"
             self.crawler.stats.inc_value(item_count_stat)
 
-        for field_name, value in item.items():
+        data = ItemAdapter(item)
+        for field_name, value in data.asdict().items():
             if skip_none_values and value is None:
                 continue
 

spidermon/contrib/scrapy/pipelines.py

@@ -1,12 +1,9 @@
-import json
-from io import BytesIO
 from collections import defaultdict
+from itemadapter import ItemAdapter
 
 from scrapy.exceptions import DropItem, NotConfigured
 from scrapy.utils.misc import load_object
-from scrapy.exporters import JsonLinesItemExporter
 from scrapy import Field, Item
-from scrapy.utils.python import to_unicode
 
 from spidermon.contrib.validation import SchematicsValidator, JSONSchemaValidator
 from spidermon.contrib.validation.jsonschema.tools import get_schema_from
@@ -136,25 +133,16 @@ def find_validators(self, item):
         return find(item.__class__) or find(Item)
 
     def _convert_item_to_dict(self, item):
-        serialized_json = BytesIO()
-        exporter = JsonLinesItemExporter(serialized_json)
-        exporter.export_item(item)
-        data = json.loads(to_unicode(serialized_json.getvalue(), exporter.encoding))
-        serialized_json.close()
-        return data
+        data = ItemAdapter(item)
+        return data.asdict()
 
     def _add_errors_to_item(self, item, errors):
-        try:
-            if self.errors_field not in item.__class__.fields:
-                item.__class__.fields[self.errors_field] = Field()
-            if self.errors_field not in item._values:
-                item[self.errors_field] = defaultdict(list)
-        except AttributeError:
-            # The item is just a dict object instead of a Scrapy.Item object
-            if self.errors_field not in item:
-                item[self.errors_field] = defaultdict(list)
+        data = ItemAdapter(item)
+        if self.errors_field not in data.keys():
+            item[self.errors_field] = defaultdict(list)
+
         for field_name, messages in errors.items():
-            item[self.errors_field][field_name] += messages
+            data[self.errors_field][field_name] += messages
 
     def _drop_item(self, item, errors):
         """