limit SQLAlchemy usage

Author: LuiggiTenorioK

autosubmit_api/builders/experiment_builder.py

@@ -1,13 +1,11 @@
 import datetime
+from autosubmit_api.logger import logger
 from autosubmit_api.builders import BaseBuilder
 from autosubmit_api.builders.configuration_facade_builder import (
     AutosubmitConfigurationFacadeBuilder,
     ConfigurationFacadeDirector,
 )
-from autosubmit_api.database import tables
-from autosubmit_api.database.common import (
-    create_autosubmit_db_engine,
-)
+from autosubmit_api.database.adapters import ExperimentDbAdapter, ExperimentDetailsDbAdapter
 from autosubmit_api.database.models import ExperimentModel
 
 
@@ -23,32 +21,26 @@ def produce_base(self, expid):
         """
         Produce basic information from the main experiment table
         """
-        with create_autosubmit_db_engine().connect() as conn:
-            result = conn.execute(
-                tables.experiment_table.select().where(
-                    tables.experiment_table.c.name == expid
-                )
-            ).one()
+        result = ExperimentDbAdapter().get_by_expid(expid)
 
         # Set new product
         self._product = ExperimentModel(
-            id=result.id,
-            name=result.name,
-            description=result.description,
-            autosubmit_version=result.autosubmit_version,
+            id=result["id"],
+            name=result["name"],
+            description=result["description"],
+            autosubmit_version=result["autosubmit_version"],
         )
 
     def produce_details(self):
         """
         Produce data from the details table
         """
         exp_id = self._product.id
-        with create_autosubmit_db_engine().connect() as conn:
-            result = conn.execute(
-                tables.details_table.select().where(
-                    tables.details_table.c.exp_id == exp_id
-                )
-            ).one_or_none()
+        result = None
+        try:
+            result = ExperimentDetailsDbAdapter().get_by_exp_id(exp_id)
+        except Exception:
+            logger.error(f"Error getting details for exp_id {exp_id}")
 
         # Set details props
         if result:
@@ -80,6 +72,7 @@ def produce_config_data(self):
             ).isoformat()
         except Exception:
             self._product.modified = None
+            logger.error(f"Error getting modified date for expid {expid}")
 
     @property
     def product(self) -> ExperimentModel:

autosubmit_api/database/__init__.py

@@ -1,28 +1,11 @@
-from sqlalchemy import Connection, Table
-from autosubmit_api.database.common import (
-    create_as_times_db_engine,
-    create_autosubmit_db_engine,
+from autosubmit_api.database.adapters import (
+    ExperimentDbAdapter,
+    ExperimentDetailsDbAdapter,
+    ExperimentStatusDbAdapter,
 )
-from autosubmit_api.database import tables
-
-
-def _create_autosubmit_db_tables(conn: Connection):
-    experiment_table: Table = tables.ExperimentTable.__table__
-    experiment_table.create(conn, checkfirst=True)
-    details_table: Table = tables.DetailsTable.__table__
-    details_table.create(conn, checkfirst=True)
-
-
-def _create_as_times_db_tables(conn: Connection):
-    experiment_status_table: Table = tables.ExperimentStatusTable.__table__
-    experiment_status_table.create(conn, checkfirst=True)
 
 
 def prepare_db():
-    with create_as_times_db_engine().connect() as conn:
-        _create_as_times_db_tables(conn)
-        conn.commit()
-
-    with create_autosubmit_db_engine().connect() as conn:
-        _create_autosubmit_db_tables(conn)
-        conn.commit()
+    ExperimentDbAdapter().create_table()
+    ExperimentDetailsDbAdapter().create_table()
+    ExperimentStatusDbAdapter().create_table()

autosubmit_api/database/adapters/__init__.py

@@ -14,13 +14,21 @@
     ExperimentStatusDbAdapter,
 )
 from autosubmit_api.database.adapters.graph_draw import ExpGraphDrawDBAdapter
-from autosubmit_api.database.adapters.join.experiment_join import ExperimentJoinDbAdapter
+from autosubmit_api.database.adapters.join.experiment_join import (
+    ExperimentJoinDbAdapter,
+)
+from autosubmit_api.database.adapters.job_packages import (
+    JobPackagesDbAdapter,
+    WrapperJobPackagesDbAdapter,
+)
 
 
 __all__ = [
     "ExperimentDbAdapter",
     "ExperimentDetailsDbAdapter",
     "ExperimentStatusDbAdapter",
     "ExpGraphDrawDBAdapter",
-    "ExperimentJoinDbAdapter"
+    "ExperimentJoinDbAdapter",
+    "JobPackagesDbAdapter",
+    "WrapperJobPackagesDbAdapter",
 ]

autosubmit_api/database/adapters/experiment.py

@@ -12,6 +12,13 @@ def __init__(self):
             db_filepath=APIBasicConfig.DB_PATH,
         )
 
+    def create_table(self):
+        """
+        Create the experiment table.
+        """
+        with self.table_manager.get_connection() as conn:
+            self.table_manager.create_table(conn)
+
     def get_all(self):
         """
         Return all experiments.

autosubmit_api/database/adapters/experiment_details.py

@@ -12,12 +12,37 @@ def __init__(self) -> None:
             db_filepath=APIBasicConfig.DB_PATH,
         )
 
+    def create_table(self):
+        """
+        Create the details table.
+        """
+        with self.table_manager.get_connection() as conn:
+            self.table_manager.create_table(conn)
+
     def delete_all(self) -> int:
+        """
+        Clear the details table.
+        """
         with self.table_manager.get_connection() as conn:
             rowcount = self.table_manager.delete_all(conn)
         return rowcount
 
     def insert_many(self, values: List[Dict[str, Any]]) -> int:
+        """
+        Insert many rows into the details table.
+        """
         with self.table_manager.get_connection() as conn:
             rowcount = self.table_manager.insert_many(conn, values)
         return rowcount
+
+    def get_by_exp_id(self, exp_id: int) -> Dict[str, Any]:
+        """
+        Get experiment details by the numerical exp_id.
+        """
+        with self.table_manager.get_connection() as conn:
+            row = conn.execute(
+                self.table_manager.table.select().where(
+                    tables.DetailsTable.exp_id == exp_id
+                )
+            ).one()
+        return row._mapping

autosubmit_api/database/adapters/experiment_status.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 import os
-from typing import Dict
+from typing import Dict, List
 from autosubmit.database.db_manager import create_db_table_manager
 from sqlalchemy import delete, insert, select
 from autosubmit_api.config.basicConfig import APIBasicConfig
@@ -15,6 +15,13 @@ def __init__(self) -> None:
             db_filepath=os.path.join(APIBasicConfig.DB_DIR, APIBasicConfig.AS_TIMES_DB),
         )
 
+    def create_table(self):
+        """
+        Create the experiment_status table.
+        """
+        with self.table_manager.get_connection() as conn:
+            self.table_manager.create_table(conn)
+
     def get_all_dict(self) -> Dict[str, str]:
         """
         Gets table experiment_status as dictionary {expid: status}
@@ -26,6 +33,18 @@ def get_all_dict(self) -> Dict[str, str]:
                 result[row.name] = row.status
         return result
 
+    def get_only_running_expids(self) -> List[str]:
+        """
+        Gets list of running experiments
+        """
+        with self.table_manager.get_connection() as conn:
+            rows = conn.execute(
+                select(self.table_manager.table).where(
+                    self.table_manager.table.c.status == "RUNNING"
+                )
+            ).all()
+        return [row.name for row in rows]
+
     def get_status(self, expid: str) -> str:
         """
         Gets the current status of one experiment

autosubmit_api/database/adapters/job_packages.py

@@ -0,0 +1,42 @@
+from typing import Dict, List
+from autosubmit.database.db_manager import create_db_table_manager
+from sqlalchemy import select
+
+from autosubmit_api.database import tables
+from autosubmit_api.persistance.experiment import ExperimentPaths
+
+
+class JobPackagesDbAdapter:
+    def __init__(self, expid: str) -> None:
+        self.expid = expid
+        self.table_manager = create_db_table_manager(
+            table=tables.JobPackageTable,
+            db_filepath=ExperimentPaths(expid).job_packages_db,
+            schema=expid,
+        )
+
+    def get_all(self) -> List[Dict[str, str]]:
+        """
+        Get all job packages.
+        """
+        with self.table_manager.get_connection() as conn:
+            rows = conn.execute(select(self.table_manager.table)).all()
+        return [row._mapping for row in rows]
+
+
+class WrapperJobPackagesDbAdapter:
+    def __init__(self, expid: str) -> None:
+        self.expid = expid
+        self.table_manager = create_db_table_manager(
+            table=tables.WrapperJobPackageTable,
+            db_filepath=ExperimentPaths(expid).job_packages_db,
+            schema=expid,
+        )
+
+    def get_all(self) -> List[Dict[str, str]]:
+        """
+        Get all job packages.
+        """
+        with self.table_manager.get_connection() as conn:
+            rows = conn.execute(select(self.table_manager.table)).all()
+        return [row._mapping for row in rows]

autosubmit_api/database/common.py

@@ -23,6 +23,8 @@ def get_postgres_engine():
 class AttachedDatabaseConnBuilder(BaseBuilder):
     """
     SQLite utility to build attached databases.
+
+    MUST BE USED ONLY FOR ADAPTERS and TESTS.
     """
 
     def __init__(self) -> None:
@@ -52,6 +54,8 @@ def product(self) -> Connection:
 def create_main_db_conn() -> Connection:
     """
     Connection with the autosubmit and as_times DDBB.
+
+    MUST BE USED ONLY FOR ADAPTERS and TESTS.
     """
     APIBasicConfig.read()
     if APIBasicConfig.DATABASE_BACKEND == "postgres":
@@ -66,6 +70,8 @@ def create_main_db_conn() -> Connection:
 def create_autosubmit_db_engine() -> Engine:
     """
     Create an engine for the autosubmit DDBB. Usually named autosubmit.db
+
+    MUST BE USED ONLY FOR ADAPTERS and TESTS.
     """
     APIBasicConfig.read()
     if APIBasicConfig.DATABASE_BACKEND == "postgres":
@@ -78,6 +84,8 @@ def create_autosubmit_db_engine() -> Engine:
 def create_as_times_db_engine() -> Engine:
     """
     Create an engine for the AS_TIMES DDBB. Usually named as_times.db
+
+    MUST BE USED ONLY FOR ADAPTERS and TESTS.
     """
 
     APIBasicConfig.read()

autosubmit_api/database/db_jobdata.py

@@ -513,7 +513,7 @@ def __init__(self, expid: str, basic_config: APIBasicConfig):
             self.db_version = self._select_pragma_version()
             # self.query_job_historic = None
             # Historic only working on DB 12 now
-            self.query_job_historic = "SELECT id, counter, job_name, created, modified, submit, start, finish, status, rowtype, ncpus, wallclock, qos, energy, date, section, member, chunk, last, platform, job_id, extra_data, nnodes, run_id FROM job_data WHERE job_name=? ORDER BY counter DESC"
+            # self.query_job_historic = "SELECT id, counter, job_name, created, modified, submit, start, finish, status, rowtype, ncpus, wallclock, qos, energy, date, section, member, chunk, last, platform, job_id, extra_data, nnodes, run_id FROM job_data WHERE job_name=? ORDER BY counter DESC"
 
             if self.db_version < DB_VERSION_SCHEMA_CHANGES:
                 try:

autosubmit_api/persistance/job_package_reader.py

@@ -1,13 +1,12 @@
 from typing import Dict, List
-from sqlalchemy import select
+from autosubmit_api.database.adapters import (
+    JobPackagesDbAdapter,
+    WrapperJobPackagesDbAdapter,
+)
 from autosubmit_api.logger import logger
-from autosubmit_api.database import tables
-from autosubmit_api.database.common import AttachedDatabaseConnBuilder
-from autosubmit_api.persistance.experiment import ExperimentPaths
 
 
 class JobPackageReader:
-
     def __init__(self, expid: str) -> None:
         self.expid = expid
         self._content: List[Dict] = []
@@ -17,23 +16,13 @@ def __init__(self, expid: str) -> None:
         self._package_to_symbol: Dict[str, str] = {}
 
     def read(self):
-        conn_builder = AttachedDatabaseConnBuilder()
-        conn_builder.attach_db(
-            ExperimentPaths(self.expid).job_packages_db, "job_packages"
-        )
-
-        with conn_builder.product as conn:
-            try:
-                statement = select(tables.JobPackageTable)
-                self._content = [x._mapping for x in conn.execute(statement).all()]
-                if len(self._content) == 0:
-                    raise Warning(
-                        "job_packages table empty, trying wrapper_job_packages"
-                    )
-            except Exception as exc:
-                logger.warning(exc)
-                statement = select(tables.WrapperJobPackageTable)
-                self._content = [x._mapping for x in conn.execute(statement).all()]
+        try:
+            self._content = JobPackagesDbAdapter(self.expid).get_all()
+            if len(self._content) == 0:
+                raise Warning("job_packages table empty, trying wrapper_job_packages")
+        except Exception as exc:
+            logger.warning(exc)
+            self._content = WrapperJobPackagesDbAdapter(self.expid).get_all()
 
         self._build_job_to_package()
         self._build_package_to_jobs()