Merge pull request #5 from KhaosResearch/v1.1

[On progress] Major changes for v1.1

Author: AmbroxMr

.gitignore

@@ -37,11 +37,22 @@ auxiliar_scripts/
 # Images
 *.png
 images/
+!static/*.png
 
 # Rasters
 *.tif
 *.jp2
 
 *.egg-info
 
-.python-version
+.python-version
+
+requirements.txt
+*.laz
+*.las
+*.xml
+
+1.1/
+1.2/
+
+*.json

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 Khaos Research
+Copyright (c) 2024 Khaos Research
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,4 +1,7 @@
-# LandCoverPy, a scalable land cover classification workflow
+# LandCoverPy, a scalable land cover/land use classification workflow
+
+![lebanon_second_level_classification](https://github.com/KhaosResearch/landcoverpy/blob/v1.1/static/lebanon_example.png)
+
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7462308.svg)](https://doi.org/10.5281/zenodo.7462308)
 
 A scalable land cover classification workflow aimed to be able to scale to cover the Mediterranean bassin.
@@ -12,15 +15,30 @@ A research article describing the methodology followed on this workflow can be f
 > Journal of Big Data 10, 91 (2023). doi: [10.1186/s40537-023-00770-z](https://doi.org/10.1186/s40537-023-00770-z)
 
 ## Installation
-From PyPi:
-`$ python -m pip install landcoverpy`
 
-From source:
-`$ make install`
+Currently, the package is not available on PyPI, so you need to install it from the source code. To do so, you can clone the repository and install it using pip:
+
+```bash
+git clone https://github.com/KhaosResearch/landcoverpy.git
+cd landcoverpy
+pip install .
+```
+
+For development purposes, you can install the package in editable mode:
+
+```bash
+pip install -e .
+```
+
+In the future, the package will be available on PyPI, so you will be able to install it using pip:
+
+```bash
+pip install landcoverpy
+```
 
 ## Usage
 
-Usage examples can be found at the [notebooks](notebooks) folder.
+An usage example can be found at the [main usage notebook](notebooks/main_usage.ipynb).
 
 ## License
 This project is licensed under the MIT license. See the [LICENSE](LICENSE) file for more info.

notebooks/explain_model.ipynb

@@ -503,7 +503,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.6"
+   "version": "3.10.12"
   },
   "vscode": {
    "interpreter": {

notebooks/examples.ipynb notebooks/main_usage.ipynbnotebooks/examples.ipynb renamed to notebooks/main_usage.ipynb

@@ -24,7 +24,9 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Postprocess generated dataset, called `dataset.csv` (which has been generated in MinIO)"
+    "### Train models using the generated dataset. \n",
+    "\n",
+    "The data will be automatically loaded from MinIO, which is also automatically saved in MinIO when using `workflow(execution_mode=ExecutionMode.TRAINING)`"
    ]
   },
   {
@@ -33,19 +35,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from landcoverpy.data_postprocessing import postprocess_dataset\n",
-    "\n",
-    "input_dataset = \"dataset.csv\"\n",
-    "land_cover_dataset = \"dataset_postprocessed.csv\"\n",
-    "\n",
-    "postprocess_dataset(input_dataset, land_cover_dataset)"
+    "from landcoverpy.model_training import train_model_land_cover\n",
+    "train_model_land_cover(n_jobs = 1)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Train model using postprocessed dataset"
+    "Now, use the generated model (also saved and loaded from MinIO) to predict a specific tile using the `ExecutionMode.LAND_COVER_PREDICTION` mode."
    ]
   },
   {
@@ -54,27 +52,40 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from landcoverpy.model_training import train_model_land_cover\n",
-    "\n",
-    "train_model_land_cover(land_cover_dataset, n_jobs = 1)"
+    "workflow(execution_mode=ExecutionMode.LAND_COVER_PREDICTION, tiles_to_predict=[\"36SYC\"])"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Get the tiles in Spain and classify them using the trained model"
+    "If you want to make second level predictions, you can train a new model that will learn from SL_PROPERTY labels from specified classes."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from landcoverpy.utilities.aoi_tiles import get_list_of_tiles_in_iberian_peninsula\n",
-    "\n",
-    "workflow(execution_mode=ExecutionMode.LAND_COVER_PREDICTION, tiles_to_predict=get_list_of_tiles_in_iberian_peninsula())"
+    "from landcoverpy.model_training import train_second_level_models\n",
+    "train_second_level_models(\"dataset.csv\", [\"bareSoil\", \"closedForest\", \"herbaceousVegetation\", \"openForest\", \"shrubland\", \"water\", \"wetland\"], n_jobs=1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Use the mode `ExecutionMode.SECOND_LEVEL_PREDICTION` to predict the second level labels of the tile using the previous land cover predictions. They will be reclassified using new models trained with the SL_PROPERTY labels in a hierarchical way."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "workflow(ExecutionMode.SECOND_LEVEL_PREDICTION,tiles_to_predict=[\"36SYC\"])"
    ]
   }
  ],
@@ -94,7 +105,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.6"
+   "version": "3.8.15"
   },
   "orig_nbformat": 4,
   "vscode": {

scripts/cloud_coverage.py

@@ -2,7 +2,6 @@
 import json
 from datetime import datetime
 from os.path import join
-from typing import List
 from pymongo.collection import Collection
 
 import pandas as pd
@@ -97,10 +96,6 @@ def _get_tiles_in_geojson(sentinel_api: SentinelAPI, geojson_path: str):
 def compute_cloud_coverage(geojson_path: str):
     """
     Computes the cloud coverage for a list of tiles included in a geojson.
-
-    Parameters:
-        countries (List[str]): List of countries included.
-
     """
 
     mongo_client = MongoConnection()

scripts/download_tiles.py

@@ -14,7 +14,7 @@
 from landcoverpy.mongo import MongoConnection
 from landcoverpy.minio import MinioConnection
 from landcoverpy.utilities.aoi_tiles import get_list_of_tiles_in_iberian_peninsula
-from landcoverpy.utilities.geometries import _group_polygons_by_tile, _kmz_to_geojson
+from landcoverpy.utilities.geometries import _group_validated_data_points_by_tile, _kmz_to_geojson, _csv_to_geojson
 from landcoverpy.utilities.utils import get_products_by_tile_and_date, get_season_dict
 from landcoverpy.workflow import  _process_tile
 
@@ -66,23 +66,20 @@ def time_training_dataset(client: Client = None):
     tiles = get_list_of_tiles_in_iberian_peninsula()
     tile = random.choice(tiles)
 
-    geojson_files = []
-    for data_class in glob(join(settings.DB_DIR, "*.kmz")):
-        if not Path.exists(Path(data_class.replace("kmz","geojson"))):
-            print(f"Parsing database to geojson: {data_class}")
-            _kmz_to_geojson(data_class)
+    data_file = settings.DB_FILE
+    if data_file.endswith(".kmz"):
+        data_file = _kmz_to_geojson(data_file)
+    if data_file.endswith(".csv"):
+        data_file = _csv_to_geojson(data_file, sep=',')
 
-    for data_class in glob(join(settings.DB_DIR, "*.geojson")):
-        print(f"Working with database {data_class}")
-        geojson_files.append(data_class)
-    polygons_per_tile = _group_polygons_by_tile(*geojson_files)
+    polygons_per_tile = _group_validated_data_points_by_tile(data_file)
 
     metadata_filename = "metadata.json"
-    metadata_filepath = join(settings.TMP_DIR, settings.LAND_COVER_MODEL_FOLDER, metadata_filename)
+    metadata_filepath = join(settings.TMP_DIR, "land-cover", metadata_filename)
 
     minio.fget_object(
         bucket_name=settings.MINIO_BUCKET_MODELS,
-        object_name=join(settings.MINIO_DATA_FOLDER_NAME, settings.LAND_COVER_MODEL_FOLDER, metadata_filename),
+        object_name=join(settings.MINIO_DATA_FOLDER_NAME, "land-cover", metadata_filename),
         file_path=metadata_filepath,
     )
 
@@ -129,11 +126,11 @@ def time_predicting_tile(client: Client = None):
 
     # For predictions, read the rasters used in "metadata.json".
     metadata_filename = "metadata.json"
-    metadata_filepath = join(settings.TMP_DIR, settings.LAND_COVER_MODEL_FOLDER, metadata_filename)
+    metadata_filepath = join(settings.TMP_DIR, "land-cover", metadata_filename)
 
     minio.fget_object(
         bucket_name=settings.MINIO_BUCKET_MODELS,
-        object_name=join(settings.MINIO_DATA_FOLDER_NAME, settings.LAND_COVER_MODEL_FOLDER, metadata_filename),
+        object_name=join(settings.MINIO_DATA_FOLDER_NAME, "land-cover", metadata_filename),
         file_path=metadata_filepath,
     )
 

scripts/execution_times.py

@@ -108,7 +108,7 @@ def jeffreys_matusita_analysis(
     )
 
     if is_forest:
-        class_column = "forest_type"
+        class_column = settings.SL_PROPERTY
         df = df.drop("class",axis=1)
     else:
         class_column = "class"

scripts/jeffreys_matusita.py

@@ -1,6 +1,6 @@
 [metadata]
 name = landcoverpy
-version = 1.0.0
+version = 1.1
 author = Antonio Manuel Burgueño Romero
 author_email = ambrbr[at]uma.es
 description = A scalable land cover classification workflow using Big Data techniques with Sentinel-2 data.
@@ -13,7 +13,7 @@ project_urls =
     Bug Tracker = https://github.com/KhaosResearch/landcoverpy/issues
 classifiers =
     Programming Language :: Python :: 3 :: Only
-    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.8
     Development Status::  5 - Production/Stable
     License :: OSI Approved :: MIT License
     Topic :: Scientific/Engineering
@@ -42,7 +42,7 @@ install_requires =
     seaborn==0.12.0
     sentinelsat==1.1.1
     Shapely==1.8.4
-    greensenti==0.5.1
+    greensenti@git+https://github.com/KhaosResearch/greensenti.git@v0.5.1
     dask==2022.04.1
     distributed==2022.4.1
     blosc==1.10.2