Modifies Models Resolution

src/main/scala/com/johnsnowlabs/client/aws/AWSGateway.scala

@@ -18,7 +18,7 @@ package com.johnsnowlabs.client.aws
 
 import com.amazonaws.auth.{AWSCredentials, AWSStaticCredentialsProvider}
 import com.amazonaws.services.pi.model.InvalidArgumentException
-import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata, PutObjectResult}
+import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata, PutObjectResult, S3Object}
 import com.amazonaws.services.s3.transfer.{Transfer, TransferManagerBuilder}
 import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
 import com.amazonaws.{AmazonClientException, AmazonServiceException, ClientConfiguration}
@@ -74,11 +74,24 @@ class AWSGateway(accessKeyId: String, secretAccessKey: String, sessionToken: Str
 
   def getMetadata(s3Path: String, folder: String, bucket: String): List[ResourceMetadata] = {
     val metaFile = getS3File(s3Path, folder, "metadata.json")
-    val obj = client.getObject(bucket, metaFile)
+    val obj = getObjectFromS3(bucket, metaFile)
     val metadata = ResourceMetadata.readResources(obj.getObjectContent)
     metadata
   }
 
+  private def getObjectFromS3(bucket: String, key: String): S3Object = {
+    try {
+      this.client.getObject(bucket, key)
+    } catch {
+      case _: AmazonClientException =>
+        val anonymousCredentialParams = CredentialParams("anonymous", "", "", "", region)
+        val awsCredentials = new AWSAnonymousCredentials
+        val credentials: Option[AWSCredentials] = awsCredentials.buildCredentials(anonymousCredentialParams)
+        val client = getAmazonS3Client(credentials)
+        client.getObject(bucket, key)
+    }
+  }
+
   def getS3File(parts: String*): String = {
     parts
       .map(part => part.stripSuffix("/"))

src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceMetadata.scala

@@ -40,7 +40,7 @@ case class ResourceMetadata
   category: Option[ResourceType] = Some(ResourceType.NOT_DEFINED),
   checksum: String = "",
   annotator: Option[String] = None
-) {
+) extends Ordered[ResourceMetadata] {
 
 
   lazy val key: String = {
@@ -64,6 +64,28 @@ case class ResourceMetadata
   private def t(time: Timestamp): String = {
     time.getTime.toString
   }
+
+  override def compare(that: ResourceMetadata): Int = {
+
+    var value: Option[Int] = None
+
+    if (this.sparkVersion == that.sparkVersion && this.libVersion == that.libVersion) {
+      value = Some(0)
+    }
+
+    if (this.sparkVersion == that.sparkVersion) {
+      if (this.libVersion.get.toFloat > that.libVersion.get.toFloat) {
+        value = Some(1)
+      } else value = Some(-1)
+    } else {
+      if (this.sparkVersion.get.toFloat > that.sparkVersion.get.toFloat) {
+        value = Some(1)
+      } else value = Some(-1)
+    }
+
+    value.get
+  }
+
 }
 
 
@@ -82,15 +104,15 @@ object ResourceMetadata {
   def resolveResource(candidates: List[ResourceMetadata],
                       request: ResourceRequest): Option[ResourceMetadata] = {
 
-    candidates
-      .filter(item => item.readyToUse
+    val compatibleCandidates = candidates
+      .filter(item => item.readyToUse && item.libVersion.isDefined && item.sparkVersion.isDefined
         && item.name == request.name
         && (request.language.isEmpty || item.language.isEmpty || request.language.get == item.language.get)
         && Version.isCompatible(request.libVersion, item.libVersion)
         && Version.isCompatible(request.sparkVersion, item.sparkVersion)
       )
-      .sortBy(item => item.time.getTime)
-      .lastOption
+
+    compatibleCandidates.sorted.lastOption
   }
 
   def readResources(file: String): List[ResourceMetadata] = {

src/main/scala/com/johnsnowlabs/util/Version.scala

@@ -18,13 +18,25 @@ package com.johnsnowlabs.util
 
 
 case class Version(parts: List[Int]) {
-  override def toString(): String = {
+
+  override def toString: String = {
     parts.mkString(".")
   }
 
   def take(n: Int): Version = {
     Version(parts.take(n))
   }
+
+  def toFloat: Float = {
+    val versionString = parts.length match {
+      case 1 => parts.head.toString
+      case 2 => f"${parts.head.toString}.${parts(1).toString}"
+      case 3 => f"${parts.head.toString}.${parts(1).toString}${parts(2).toString}"
+      case _ => throw new UnsupportedOperationException(f"Cannot cast to float version ${this.toString()}")
+    }
+    versionString.toFloat
+  }
+
 }
 
 object Version {

src/test/resources/resource-downloader/test_bert_v2_newest.json

@@ -0,0 +1,9 @@
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 2, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2019-08-24T18:30:27.398Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 4, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2020-02-01T17:52:37.778Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 6, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2020-08-25T07:25:36.670Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "nl", "libVersion": {"parts": [3, 2, 2]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T07:28:37.658Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "sv", "libVersion": {"parts": [3, 2, 2]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T07:27:51.555Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "es", "libVersion": {"parts": [3, 2, 2]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T07:27:11.885Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "ro", "libVersion": {"parts": [3, 2, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-13T11:47:15.237Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [3, 3, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2021-11-14T17:52:37.778Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [3, 3, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-10-14T17:52:37.778Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}

src/test/resources/resource-downloader/test_bert_v3_newest.json

@@ -0,0 +1,9 @@
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 2, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2019-08-24T18:30:27.398Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 4, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2020-02-01T17:52:37.778Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 6, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2020-08-25T07:25:36.670Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "nl", "libVersion": {"parts": [3, 2, 2]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T07:28:37.658Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "sv", "libVersion": {"parts": [3, 2, 2]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T07:27:51.555Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "es", "libVersion": {"parts": [3, 2, 2]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T07:27:11.885Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "ro", "libVersion": {"parts": [3, 2, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-13T11:47:15.237Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [3, 3, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2021-10-14T17:52:37.778Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [3, 3, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-11-14T17:52:37.778Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}

src/test/resources/resource-downloader/test_example1.json

@@ -0,0 +1,8 @@
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 2, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2019-08-24T18:30:27.398Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 4, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2020-02-01T17:52:37.778Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 6, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2020-08-25T07:25:36.670Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "nl", "libVersion": {"parts": [3, 2, 2]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T07:28:37.658Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "sv", "libVersion": {"parts": [3, 2, 2]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T07:27:51.555Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "es", "libVersion": {"parts": [3, 2, 2]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T07:27:11.885Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "ro", "libVersion": {"parts": [3, 2, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-13T11:47:15.237Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 4, 5]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-11-14T17:52:37.778Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}

src/test/resources/resource-downloader/test_models_same_time.json

@@ -0,0 +1,9 @@
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 2, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2019-08-24T18:30:27.398Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 4, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2020-02-01T17:52:37.778Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [2, 6, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2020-08-25T07:25:36.670Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "nl", "libVersion": {"parts": [3, 2, 2]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T07:28:37.658Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "sv", "libVersion": {"parts": [3, 2, 2]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T07:27:51.555Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "es", "libVersion": {"parts": [3, 2, 2]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T07:27:11.885Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "ro", "libVersion": {"parts": [3, 2, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-13T11:47:15.237Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [3, 3, 0]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2021-11-14T17:52:37.778Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}
+{"name": "bert_base_cased", "language": "en", "libVersion": {"parts": [3, 3, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-11-14T17:52:37.778Z", "isZipped": true, "category": "ml", "checksum": "", "annotator": "BertEmbeddings"}

src/test/resources/resource-downloader/test_v2_latest_date.json

@@ -0,0 +1,8 @@
+{"name": "explain_document_dl", "language": "en", "libVersion": {"parts": [3, 0, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-03-23T04:21:08.265Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name": "recognize_entities_dl", "language": "en", "libVersion": {"parts": [3, 0, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-03-23T04:27:27.254Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name": "recognize_entities_bert", "language": "en", "libVersion": {"parts": [3, 0, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-03-23T04:31:43.583Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name": "explain_document_dl", "language": "en", "libVersion": {"parts": [3, 1, 1]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2021-09-06T21:52:55.425Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name": "explain_document_dl", "language": "en", "libVersion": {"parts": [3, 1, 1]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-10-07T18:04:49.759Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name": "explain_document_dl", "language": "en", "libVersion": {"parts": [3, 1, 3]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2021-09-07T19:43:32.023Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name": "explain_document_dl", "language": "en", "libVersion": {"parts": [3, 1, 3]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T20:25:43.759Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name":"explain_document_dl", "language": "en", "libVersion": {"parts": [3, 1, 3]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2021-11-10T20:25:43.759Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}

src/test/resources/resource-downloader/test_with_spark_3.2.json

@@ -0,0 +1,9 @@
+{"name": "explain_document_dl", "language": "en", "libVersion": {"parts": [3, 0, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-03-23T04:21:08.265Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name": "recognize_entities_dl", "language": "en", "libVersion": {"parts": [3, 0, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-03-23T04:27:27.254Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name": "recognize_entities_bert", "language": "en", "libVersion": {"parts": [3, 0, 0]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-03-23T04:31:43.583Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name": "explain_document_dl", "language": "en", "libVersion": {"parts": [3, 1, 1]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2021-09-06T21:52:55.425Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name": "explain_document_dl", "language": "en", "libVersion": {"parts": [3, 1, 1]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-10-07T18:04:49.759Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name": "explain_document_dl", "language": "en", "libVersion": {"parts": [3, 1, 3]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2021-09-07T19:43:32.023Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name": "explain_document_dl", "language": "en", "libVersion": {"parts": [3, 1, 3]}, "sparkVersion": {"parts": [3, 0]}, "readyToUse": true, "time": "2021-09-07T20:25:43.759Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name":"explain_document_dl", "language": "en", "libVersion": {"parts": [3, 1, 3]}, "sparkVersion": {"parts": [2, 4]}, "readyToUse": true, "time": "2021-11-10T20:25:43.759Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}
+{"name":"explain_document_dl", "language": "en", "libVersion": {"parts": [3, 4, 0]}, "sparkVersion": {"parts": [3, 2]}, "readyToUse": true, "time": "2021-12-10T20:25:43.759Z", "isZipped": true, "category": "pl", "checksum": "", "annotator": "PipelineModel"}

src/test/scala/com/johnsnowlabs/nlp/annotators/er/EntityRulerTest.scala

@@ -506,7 +506,7 @@ class EntityRulerTest extends AnyFlatSpec with SparkSessionTest {
     AssertAnnotations.assertFields(expectedEntities, actualEntities)
   }
 
-  it should "serialize and deserialize a model without storage" in {
+  it should "serialize and deserialize a model without storage" taggedAs FastTest in {
     val textDataSet = Seq("Lord Eddard Stark was the head of House Stark").toDS.toDF("text")
     tokenizer.setExceptions(Array("Eddard Stark"))
     val entityRuler = new EntityRulerApproach()

src/test/scala/com/johnsnowlabs/nlp/pretrained/CloudTestResources.scala

@@ -31,6 +31,15 @@ object CloudTestResources {
     new Timestamp(50)
   )
 
+  val name_en_123_34_new = new ResourceMetadata(
+    "name",
+    Some("en"),
+    Some(Version(1, 2, 3)),
+    Some(Version(3, 4)),
+    true,
+    new Timestamp(50)
+  )
+
   val name_en_12_34_old = new ResourceMetadata(
     "name",
     Some("en"),
@@ -90,5 +99,5 @@ object CloudTestResources {
     true,
     new Timestamp(1)
   )
-  val all = List(name_en_123_345_new, name_en_12_34_old, name_en_old, name_en_new_disabled, name_de)
+  val all = List(name_en_123_345_new, name_en_12_34_old, name_en_old, name_en_new_disabled, name_de, name_en_123_34_new)
 }

src/test/scala/com/johnsnowlabs/nlp/pretrained/MockResourceDownloader.scala

@@ -0,0 +1,23 @@
+package com.johnsnowlabs.nlp.pretrained
+
+/***
+ * Mock class of the ResourceDownloader, that will use a local file instead of downloading the metadata.json from
+ * S3.
+ */
+class MockResourceDownloader(resourcePath: String) extends ResourceDownloader {
+
+  def download(request: ResourceRequest): Option[String] = {
+    Some("MockResourceDownloader")
+  }
+
+  def getDownloadSize(request: ResourceRequest): Option[Long] = {
+    Some(0)
+  }
+
+  def clearCache(request: ResourceRequest): Unit = {}
+
+  val resources: List[ResourceMetadata] = ResourceMetadata.readResources(resourcePath)
+
+  def downloadMetadataIfNeed(folder: String): List[ResourceMetadata] = resources
+
+}