Feature/batch lru update

conan/internal/cache/cache.py

@@ -269,14 +269,14 @@ def assign_rrev(self, layout: RecipeLayout):
     def get_recipe_lru(self, ref):
         return self._db.get_recipe_lru(ref)
 
-    def update_recipe_lru(self, ref):
-        self._db.update_recipe_lru(ref)
+    def update_recipes_lru(self, refs):
+        self._db.update_recipes_lru(refs)
 
     def get_package_lru(self, pref):
         return self._db.get_package_lru(pref)
 
-    def update_package_lru(self, pref):
-        self._db.update_package_lru(pref)
+    def update_packages_lru(self, prefs):
+        self._db.update_packages_lru(prefs)
 
     def path_to_ref(self, path):
         try:

conan/internal/cache/db/cache_database.py

@@ -40,11 +40,11 @@ def get_recipe_lru(self, ref):
     def get_package_lru(self, pref: PkgReference):
         return self._packages.get(pref)["lru"]
 
-    def update_recipe_lru(self, ref):
-        self._recipes.update_lru(ref)
+    def update_recipes_lru(self, refs):
+        self._recipes.update_lru(refs)
 
-    def update_package_lru(self, pref):
-        self._packages.update_lru(pref)
+    def update_packages_lru(self, prefs):
+        self._packages.update_lru(prefs)
 
     def remove_recipe(self, ref: RecipeReference):
         # Removing the recipe must remove all the package binaries too from DB

conan/internal/cache/db/packages_table.py

@@ -100,17 +100,19 @@ def update_timestamp(self, pref: PkgReference, path: str, build_id: str):
             except sqlite3.IntegrityError:
                 raise ConanReferenceAlreadyExistsInDB(f"Reference '{repr(pref)}' already exists")
 
-    def update_lru(self, pref):
-        assert pref.revision is not None
+    def update_lru(self, prefs):
         # TODO: InstallGraph is dropping the pref.timestamp, cannot be checked here yet
         # assert pref.timestamp is not None, f"PREF _TIMESSTAMP IS NONE {repr(pref)}"
-        where_clause = self._where_clause(pref)
+        params = [(str(pref.ref), pref.ref.revision, pref.package_id, pref.revision)
+                  for pref in prefs]
+        where_clause = (f"{self.columns.reference} = ? AND {self.columns.rrev} = ? "
+                        f"AND {self.columns.pkgid} = ? AND {self.columns.prev} = ?")
         lru = timestamp_now()
         query = f"UPDATE {self.table_name} " \
                 f"SET {self.columns.lru} = '{lru}' " \
                 f"WHERE {where_clause};"
         with self.db_connection() as conn:
-            conn.execute(query)
+            conn.executemany(query, params)
 
     def remove_build_id(self, pref):
         where_clause = self._where_clause(pref)

conan/internal/cache/db/recipes_table.py

@@ -60,16 +60,15 @@ def update_timestamp(self, ref: RecipeReference):
         with self.db_connection() as conn:
             conn.execute(query)
 
-    def update_lru(self, ref):
-        assert ref.revision is not None
-        assert ref.timestamp is not None
-        where_clause = self._where_clause(ref)
+    def update_lru(self, refs):
+        params = [(str(ref), ref.revision) for ref in refs]
+        where_clause = f"{self.columns.reference} = ? AND {self.columns.rrev} = ?"
         lru = timestamp_now()
         query = f"UPDATE {self.table_name} " \
                 f"SET {self.columns.lru} = '{lru}' " \
                 f"WHERE {where_clause};"
         with self.db_connection() as conn:
-            conn.execute(query)
+            conn.executemany(query, params)
 
     def remove(self, ref: RecipeReference):
         where_clause = self._where_clause(ref)

conan/internal/graph/graph.py

@@ -12,7 +12,7 @@
 RECIPE_NEWER = "Newer"  # The local recipe is  modified and newer timestamp than server
 RECIPE_NOT_IN_REMOTE = "Not in remote"
 RECIPE_UPDATEABLE = "Update available"  # The update of recipe is available (only in conan info)
-RECIPE_NO_REMOTE = "No remote"
+# These recipes do not have a full reference, not in the cache
 RECIPE_EDITABLE = "Editable"
 RECIPE_CONSUMER = "Consumer"  # A conanfile from the user
 RECIPE_VIRTUAL = "Cli"  # A virtual conanfile (dynamic in memory conanfile)

conan/internal/graph/graph_builder.py

@@ -4,7 +4,7 @@
 from conan.internal.methods import run_configure_method
 from conan.internal.model.recipe_ref import ref_matches
 from conan.internal.graph.graph import DepsGraph, Node, CONTEXT_HOST, \
-    CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL, RECIPE_EDITABLE
+    CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL, RECIPE_EDITABLE, RECIPE_CONSUMER
 from conan.internal.graph.graph import RECIPE_PLATFORM
 from conan.internal.graph.graph_error import (GraphLoopError, GraphConflictError, GraphMissingError,
                                               GraphError)
@@ -67,6 +67,10 @@ def load_graph(self, root_node, profile_host, profile_build, graph_lock=None):
         except GraphError as e:
             dep_graph.error = e
         dep_graph.resolved_ranges = self._resolver.resolved_ranges
+        refs = set(n.ref for n in dep_graph.nodes
+                   if n.recipe not in (RECIPE_VIRTUAL, RECIPE_EDITABLE, RECIPE_CONSUMER,
+                                       RECIPE_PLATFORM))
+        self._cache.update_recipes_lru(refs)
         return dep_graph
 
     def _expand_require(self, require, node, graph, profile_host, profile_build, graph_lock):

conan/internal/graph/installer.py

@@ -253,12 +253,17 @@ def install(self, deps_graph, remotes, install_order=None):
         handled_count = 1
 
         self._download_bulk(install_order)
+        prefs_lru = []
         for level in install_order:
             for install_reference in level:
                 for package in install_reference.packages.values():
                     self._install_source(package.nodes[0], remotes)
                     self._handle_package(package, install_reference, handled_count, package_count)
                     handled_count += 1
+                    if package.binary == BINARY_CACHE:
+                        prefs_lru.append(package.nodes[0].pref)
+
+        self._cache.update_packages_lru(prefs_lru)
 
         MockInfoProperty.message()
 
@@ -329,7 +334,6 @@ def _handle_package(self, package, install_reference, handled_count, total_count
             if package.binary == BINARY_CACHE:
                 node = package.nodes[0]
                 pref = node.pref
-                self._cache.update_package_lru(pref)
                 assert node.prev, "PREV for %s is None" % str(pref)
                 msg = f'Already installed! ({handled_count} of {total_count})'
                 node.conanfile.output.success(msg)

conan/internal/graph/proxy.py

@@ -49,8 +49,6 @@ def _get_recipe(self, reference, remotes, update, check_update):
             status = RECIPE_DOWNLOADED
             return layout, status, remote
 
-        self._cache.update_recipe_lru(ref)
-
         # TODO: cache2.0: check with new --update flows
         # TODO: If the revision is given, then we don't need to check for updates?
         if not (check_update or should_update_reference(reference, update)):

test/performance/test_db_performance.py

@@ -0,0 +1,74 @@
+import os
+import time
+
+import pytest
+
+from conan.api.model import RecipeReference
+from conan.internal.cache.db.cache_database import CacheDatabase
+from conan.test.utils.test_files import temp_folder
+
+
+@pytest.mark.skip(reason="This is a performance test, skip for normal runs")
+def test_db_performance():
+    f = temp_folder()
+    # f = r"C:\conan_tests\tmp_o0846cmconans\path with spaces"
+    print("Tempt folder: ", f)
+    db = CacheDatabase(os.path.join(f, "mytest.sqlite"))
+
+    num_refs = 1000
+    splits = 10
+    for num_split in range(10):
+        init = time.time()
+        for i in range(int(num_refs / splits)):
+            index = num_split * int(num_refs / splits) + i
+            ref = RecipeReference.loads(f"pkg/1.{index}#rev1%1")
+            path = os.path.join(f, f"folder{index}")
+            db.create_recipe(path, ref)
+        creation_time = time.time() - init
+        print(f"Creation time {num_split}:", creation_time)
+        print("    Avg:", creation_time/num_refs)
+
+    experiments = 10
+    texp = time.time()
+    for experiment in range(experiments):
+        ret = db.list_references()
+        assert len(ret) == num_refs
+    exp_time = time.time() - texp
+    print("SEARCH RECIPES time:", exp_time)
+    print("    Avg:", exp_time / experiments)
+
+    texp = time.time()
+    specific_ref = RecipeReference.loads(f"pkg/1.1#rev1%1")
+    for experiment in range(experiments):
+        db.get_recipe(specific_ref)
+    exp_time = time.time() - texp
+    print("GET RECIPE time:", exp_time)
+    print("    Avg:", exp_time / experiments)
+
+    texp = time.time()
+    specific_ref = RecipeReference.loads(f"pkg/1.1#rev1%1")
+    for experiment in range(experiments):
+        db.get_latest_recipe(specific_ref)
+    exp_time = time.time() - texp
+    print("GET LATEST RECIPE time:", exp_time)
+    print("    Avg:", exp_time / experiments)
+
+    texp = time.time()
+    for experiment in range(experiments):
+        db.update_recipes_lru([specific_ref])
+    exp_time = time.time() - texp
+    print("UPDATE LRU:", exp_time)
+    print("    Avg:", exp_time / experiments)
+
+    updates = 50
+    texp = time.time()
+    for experiment in range(experiments):
+        refs = [RecipeReference.loads(f"pkg/1.{index}#rev1%1") for index in range(updates)]
+        db.update_recipes_lru(refs)
+    exp_time = time.time() - texp
+    print("UPDATE LRU BATCH:", exp_time)
+    print("    Avg:", exp_time / experiments)
+    print("    Avg:", exp_time / (experiments * updates))
+
+
+