Spark NLP 6.1.0 Release

.github/workflows/create_search_index.yml

@@ -11,7 +11,8 @@ concurrency:
 
 jobs:
   jekyll:
-    runs-on: ubuntu-latest
+    runs-on: RAM32GB
+    timeout-minutes: 600
     environment: jekyll
     steps:
       - uses: actions/checkout@v2
@@ -49,6 +50,8 @@ jobs:
           ELASTICSEARCH_INDEX_NAME: ${{ secrets.ELASTICSEARCH_INDEX_NAME }}
           SEARCH_ORIGIN: ${{ secrets.SEARCH_ORIGIN }}
           ORIGIN: ${{ secrets.ORIGIN }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.MODELS_PUBLIC_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.MODELS_SECRET_KEY }}
         working-directory: docs
         run: |
           bundle exec jekyll build --incremental
@@ -62,6 +65,8 @@ jobs:
           ELASTICSEARCH_INDEX_NAME: ${{ secrets.ELASTICSEARCH_INDEX_NAME }}
           SEARCH_ORIGIN: ${{ secrets.SEARCH_ORIGIN }}
           ORIGIN: ${{ secrets.ORIGIN }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.MODELS_PUBLIC_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.MODELS_SECRET_KEY }}
         working-directory: docs
         run: |
           rm -f .jekyll-metadata

CHANGELOG

@@ -1,3 +1,19 @@
+=======
+6.1.0
+=======
+---------------------------
+New Features & Enhancements
+---------------------------
+
+* [SPARKNLP-1189] Introducing Phi4
+* [SPARKNLP-1259] Introducing Reader2Doc Annotator
+* [SPARKNLP-1194] Upgrade jsl-llamacpp to newest version
+
+---------
+Bug Fixes
+---------
+* Fix HuggingFace_OpenVINO_in_Spark_NLP_Qwen2VL.ipynb
+
 =======
 6.0.5
 =======

README.md

@@ -63,7 +63,7 @@ $ java -version
 $ conda create -n sparknlp python=3.7 -y
 $ conda activate sparknlp
 # spark-nlp by default is based on pyspark 3.x
-$ pip install spark-nlp==6.0.5 pyspark==3.3.1
+$ pip install spark-nlp==6.1.0 pyspark==3.3.1
 ```
 
 In Python console or Jupyter `Python3` kernel:
@@ -129,11 +129,11 @@ For a quick example of using pipelines and models take a look at our official [d
 
 ### Apache Spark Support
 
-Spark NLP *6.0.5* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x
+Spark NLP *6.1.0* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x
 
 | Spark NLP | Apache Spark 3.5.x | Apache Spark 3.4.x | Apache Spark 3.3.x | Apache Spark 3.2.x | Apache Spark 3.1.x | Apache Spark 3.0.x | Apache Spark 2.4.x | Apache Spark 2.3.x |
 |-----------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|
-| 6.0.x     | YES                | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
+| 6.x.x and up     | YES                | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
 | 5.5.x     | YES                | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
 | 5.4.x     | YES                | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
 | 5.3.x     | YES                | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
@@ -159,24 +159,25 @@ Find out more about 4.x `SparkNLP` versions in our official [documentation](http
 
 ### Databricks Support
 
-Spark NLP 6.0.5 has been tested and is compatible with the following runtimes:
+Spark NLP 6.1.0 has been tested and is compatible with the following runtimes:
 
 | **CPU**            | **GPU**            |
 |--------------------|--------------------|
 | 14.1 / 14.1 ML     | 14.1 ML & GPU      |
 | 14.2 / 14.2 ML     | 14.2 ML & GPU      |
 | 14.3 / 14.3 ML     | 14.3 ML & GPU      |
 | 15.0 / 15.0 ML     | 15.0 ML & GPU      |
-| 15.1 / 15.0 ML     | 15.1 ML & GPU      |
-| 15.2 / 15.0 ML     | 15.2 ML & GPU      |
-| 15.3 / 15.0 ML     | 15.3 ML & GPU      |
-| 15.4 / 15.0 ML     | 15.4 ML & GPU      |
+| 15.1 / 15.1 ML     | 15.1 ML & GPU      |
+| 15.2 / 15.2 ML     | 15.2 ML & GPU      |
+| 15.3 / 15.3 ML     | 15.3 ML & GPU      |
+| 15.4 / 15.4 ML     | 15.4 ML & GPU      |
+| 16.4 / 16.4 ML     | 16.4 ML & GPU      |
 
 We are compatible with older runtimes. For a full list check databricks support in our official [documentation](https://sparknlp.org/docs/en/install#databricks-support)
 
 ### EMR Support
 
-Spark NLP 6.0.5 has been tested and is compatible with the following EMR releases:
+Spark NLP 6.1.0 has been tested and is compatible with the following EMR releases:
 
 | **EMR Release**    |
 |--------------------|

build.sbt

@@ -6,7 +6,7 @@ name := getPackageName(is_silicon, is_gpu, is_aarch64)
 
 organization := "com.johnsnowlabs.nlp"
 
-version := "6.0.5"
+version := "6.1.0"
 
 (ThisBuild / scalaVersion) := scalaVer
 

conda/meta.yaml

@@ -1,13 +1,13 @@
 {% set name = "spark-nlp" %}
-{% set version = "6.0.5" %}
+{% set version = "6.1.0" %}
 
 package:
   name: {{ name|lower }}
   version: {{ version }}
 
 source:
   url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/spark_nlp-{{ version }}.tar.gz
-  sha256: 0610b5b78db44b934764e1a4bdfe2fd425e6e6bc03104aeefc83f0e5c9e2808e
+  sha256: 1356e0839868a6c4b5f6befad7e937e43864b57e4a0feb168b06395906136a27
 
 build:
   noarch: python

docs/Gemfile

@@ -11,6 +11,9 @@ gem "webrick"
 
 gem "jekyll", "~> 3.9"
 
+gem "aws-sdk-s3", "~>1"
+
+
 group "jekyll-plugins" do
     gem "jekyll-incremental", "0.1.0", path: "_plugins/jekyll-incremental"
 end

docs/Gemfile.lock

@@ -15,6 +15,25 @@ GEM
       zeitwerk (~> 2.2, >= 2.2.2)
     addressable (2.8.1)
       public_suffix (>= 2.0.2, < 6.0)
+    aws-eventstream (1.4.0)
+    aws-partitions (1.1126.0)
+    aws-sdk-core (3.226.2)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.992.0)
+      aws-sigv4 (~> 1.9)
+      base64
+      jmespath (~> 1, >= 1.6.1)
+      logger
+    aws-sdk-kms (1.106.0)
+      aws-sdk-core (~> 3, >= 3.225.0)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-s3 (1.192.0)
+      aws-sdk-core (~> 3, >= 3.225.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.5)
+    aws-sigv4 (1.12.1)
+      aws-eventstream (~> 1, >= 1.0.2)
+    base64 (0.3.0)
     coffee-script (2.4.1)
       coffee-script-source
       execjs
@@ -233,6 +252,7 @@ GEM
       gemoji (~> 3.0)
       html-pipeline (~> 2.2)
       jekyll (>= 3.0, < 5.0)
+    jmespath (1.6.2)
     kramdown (2.3.2)
       rexml
     kramdown-parser-gfm (1.1.0)
@@ -241,6 +261,7 @@ GEM
     listen (3.8.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
+    logger (1.7.0)
     mercenary (0.3.6)
     mini_portile2 (2.8.1)
     minima (2.5.1)
@@ -301,6 +322,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  aws-sdk-s3 (~> 1)
   elasticsearch (~> 7.10)
   github-pages (= 227)
   jekyll (~> 3.9)

docs/_plugins/search_index.rb

@@ -5,7 +5,9 @@
 require 'date'
 require 'elasticsearch'
 require 'nokogiri'
+require 'aws-sdk-s3'
 
+BUCKET_NAME="pypi.johnsnowlabs.com"
 SEARCH_URL = (ENV["SEARCH_ORIGIN"] || 'https://search.modelshub.johnsnowlabs.com') + '/'
 ELASTICSEARCH_INDEX_NAME = ENV["ELASTICSEARCH_INDEX_NAME"] || 'models'
 
@@ -17,6 +19,18 @@
 
 $remote_editions = Set.new
 
+def upload_file_to_s3_bucket(file_path)
+  s3 = Aws::S3::Client.new(region: 'eu-west-1')
+  object_key = "public/models.json"
+  begin
+    s3.put_object(bucket: BUCKET_NAME, key: object_key, body: File.open(file_path, 'rb'), acl: 'public-read')
+    puts "File uploaded successfully to #{BUCKET_NAME}/#{object_key}"
+
+  rescue Aws::S3::Errors::ServiceError => e
+    puts "Failed to upload file: #{e.message}"
+  end
+end
+
 class Version < Array
   def initialize name
     m = /(\d+\.\d+)\z/.match(name)
@@ -252,7 +266,7 @@ def initialize(client)
 
   def index(id, data)
     @buffer << { update: { _id: id, data: {doc: data, doc_as_upsert: true}} }
-    self.execute if @buffer.length >= 100
+    self.execute if @buffer.length >= 500
   end
 
   def execute
@@ -578,9 +592,14 @@ def is_latest?(group, model)
     models_references_json = backup_references_data.merge(models_references_json)
   end
 
-  filename = File.join(site.config['destination'], 'models.json')
+  filename = File.join(site.config['destination'], 'backup-modelss3.json')
+
   File.write(filename, models_json.values.to_json)
   File.write(backup_filename, models_json.to_json)
+  # models.json moved to pypi s3 bucket
+  upload_file_to_s3_bucket(filename)
+
+  File.delete(filename)
 
   benchmarking_filename = File.join(site.config['destination'], 'benchmarking.json')
   File.write(benchmarking_filename, models_benchmarking_json.to_json)

docs/api/com/index.html

@@ -3,9 +3,9 @@
         <head>
           <meta http-equiv="X-UA-Compatible" content="IE=edge" />
           <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
-          <title>Spark NLP 6.0.5 ScalaDoc  - com</title>
-          <meta name="description" content="Spark NLP 6.0.5 ScalaDoc - com" />
-          <meta name="keywords" content="Spark NLP 6.0.5 ScalaDoc com" />
+          <title>Spark NLP 6.1.0-rc1 ScalaDoc  - com</title>
+          <meta name="description" content="Spark NLP 6.1.0 - rc1 ScalaDoc - com" />
+          <meta name="keywords" content="Spark NLP 6.1.0 rc1 ScalaDoc com" />
           <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
 
 
@@ -28,7 +28,7 @@
         </head>
         <body>
       <div id="search">
-        <span id="doc-title">Spark NLP 6.0.5 ScalaDoc<span id="doc-version"></span></span>
+        <span id="doc-title">Spark NLP 6.1.0-rc1 ScalaDoc<span id="doc-version"></span></span>
         <span class="close-results"><span class="left">&lt;</span> Back</span>
         <div id="textfilter">
           <span class="input">

docs/api/com/johnsnowlabs/client/CloudClient.html

@@ -3,9 +3,9 @@
         <head>
           <meta http-equiv="X-UA-Compatible" content="IE=edge" />
           <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
-          <title>Spark NLP 6.0.5 ScalaDoc  - com.johnsnowlabs.client.CloudClient</title>
-          <meta name="description" content="Spark NLP 6.0.5 ScalaDoc - com.johnsnowlabs.client.CloudClient" />
-          <meta name="keywords" content="Spark NLP 6.0.5 ScalaDoc com.johnsnowlabs.client.CloudClient" />
+          <title>Spark NLP 6.1.0-rc1 ScalaDoc  - com.johnsnowlabs.client.CloudClient</title>
+          <meta name="description" content="Spark NLP 6.1.0 - rc1 ScalaDoc - com.johnsnowlabs.client.CloudClient" />
+          <meta name="keywords" content="Spark NLP 6.1.0 rc1 ScalaDoc com.johnsnowlabs.client.CloudClient" />
           <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
 
 
@@ -28,7 +28,7 @@
         </head>
         <body>
       <div id="search">
-        <span id="doc-title">Spark NLP 6.0.5 ScalaDoc<span id="doc-version"></span></span>
+        <span id="doc-title">Spark NLP 6.1.0-rc1 ScalaDoc<span id="doc-version"></span></span>
         <span class="close-results"><span class="left">&lt;</span> Back</span>
         <div id="textfilter">
           <span class="input">

docs/api/com/johnsnowlabs/client/CloudManager.html

@@ -3,9 +3,9 @@
         <head>
           <meta http-equiv="X-UA-Compatible" content="IE=edge" />
           <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
-          <title>Spark NLP 6.0.5 ScalaDoc  - com.johnsnowlabs.client.CloudManager</title>
-          <meta name="description" content="Spark NLP 6.0.5 ScalaDoc - com.johnsnowlabs.client.CloudManager" />
-          <meta name="keywords" content="Spark NLP 6.0.5 ScalaDoc com.johnsnowlabs.client.CloudManager" />
+          <title>Spark NLP 6.1.0-rc1 ScalaDoc  - com.johnsnowlabs.client.CloudManager</title>
+          <meta name="description" content="Spark NLP 6.1.0 - rc1 ScalaDoc - com.johnsnowlabs.client.CloudManager" />
+          <meta name="keywords" content="Spark NLP 6.1.0 rc1 ScalaDoc com.johnsnowlabs.client.CloudManager" />
           <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
 
 
@@ -28,7 +28,7 @@
         </head>
         <body>
       <div id="search">
-        <span id="doc-title">Spark NLP 6.0.5 ScalaDoc<span id="doc-version"></span></span>
+        <span id="doc-title">Spark NLP 6.1.0-rc1 ScalaDoc<span id="doc-version"></span></span>
         <span class="close-results"><span class="left">&lt;</span> Back</span>
         <div id="textfilter">
           <span class="input">

docs/api/com/johnsnowlabs/client/CloudResources$.html

@@ -3,9 +3,9 @@
         <head>
           <meta http-equiv="X-UA-Compatible" content="IE=edge" />
           <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
-          <title>Spark NLP 6.0.5 ScalaDoc  - com.johnsnowlabs.client.CloudResources</title>
-          <meta name="description" content="Spark NLP 6.0.5 ScalaDoc - com.johnsnowlabs.client.CloudResources" />
-          <meta name="keywords" content="Spark NLP 6.0.5 ScalaDoc com.johnsnowlabs.client.CloudResources" />
+          <title>Spark NLP 6.1.0-rc1 ScalaDoc  - com.johnsnowlabs.client.CloudResources</title>
+          <meta name="description" content="Spark NLP 6.1.0 - rc1 ScalaDoc - com.johnsnowlabs.client.CloudResources" />
+          <meta name="keywords" content="Spark NLP 6.1.0 rc1 ScalaDoc com.johnsnowlabs.client.CloudResources" />
           <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
 
 
@@ -28,7 +28,7 @@
         </head>
         <body>
       <div id="search">
-        <span id="doc-title">Spark NLP 6.0.5 ScalaDoc<span id="doc-version"></span></span>
+        <span id="doc-title">Spark NLP 6.1.0-rc1 ScalaDoc<span id="doc-version"></span></span>
         <span class="close-results"><span class="left">&lt;</span> Back</span>
         <div id="textfilter">
           <span class="input">

docs/api/com/johnsnowlabs/client/CloudStorage.html

@@ -3,9 +3,9 @@
         <head>
           <meta http-equiv="X-UA-Compatible" content="IE=edge" />
           <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
-          <title>Spark NLP 6.0.5 ScalaDoc  - com.johnsnowlabs.client.CloudStorage</title>
-          <meta name="description" content="Spark NLP 6.0.5 ScalaDoc - com.johnsnowlabs.client.CloudStorage" />
-          <meta name="keywords" content="Spark NLP 6.0.5 ScalaDoc com.johnsnowlabs.client.CloudStorage" />
+          <title>Spark NLP 6.1.0-rc1 ScalaDoc  - com.johnsnowlabs.client.CloudStorage</title>
+          <meta name="description" content="Spark NLP 6.1.0 - rc1 ScalaDoc - com.johnsnowlabs.client.CloudStorage" />
+          <meta name="keywords" content="Spark NLP 6.1.0 rc1 ScalaDoc com.johnsnowlabs.client.CloudStorage" />
           <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
 
 
@@ -28,7 +28,7 @@
         </head>
         <body>
       <div id="search">
-        <span id="doc-title">Spark NLP 6.0.5 ScalaDoc<span id="doc-version"></span></span>
+        <span id="doc-title">Spark NLP 6.1.0-rc1 ScalaDoc<span id="doc-version"></span></span>
         <span class="close-results"><span class="left">&lt;</span> Back</span>
         <div id="textfilter">
           <span class="input">

docs/api/com/johnsnowlabs/client/aws/AWSAnonymousCredentials.html

@@ -3,9 +3,9 @@
         <head>
           <meta http-equiv="X-UA-Compatible" content="IE=edge" />
           <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
-          <title>Spark NLP 6.0.5 ScalaDoc  - com.johnsnowlabs.client.aws.AWSAnonymousCredentials</title>
-          <meta name="description" content="Spark NLP 6.0.5 ScalaDoc - com.johnsnowlabs.client.aws.AWSAnonymousCredentials" />
-          <meta name="keywords" content="Spark NLP 6.0.5 ScalaDoc com.johnsnowlabs.client.aws.AWSAnonymousCredentials" />
+          <title>Spark NLP 6.1.0-rc1 ScalaDoc  - com.johnsnowlabs.client.aws.AWSAnonymousCredentials</title>
+          <meta name="description" content="Spark NLP 6.1.0 - rc1 ScalaDoc - com.johnsnowlabs.client.aws.AWSAnonymousCredentials" />
+          <meta name="keywords" content="Spark NLP 6.1.0 rc1 ScalaDoc com.johnsnowlabs.client.aws.AWSAnonymousCredentials" />
           <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
 
 
@@ -28,7 +28,7 @@
         </head>
         <body>
       <div id="search">
-        <span id="doc-title">Spark NLP 6.0.5 ScalaDoc<span id="doc-version"></span></span>
+        <span id="doc-title">Spark NLP 6.1.0-rc1 ScalaDoc<span id="doc-version"></span></span>
         <span class="close-results"><span class="left">&lt;</span> Back</span>
         <div id="textfilter">
           <span class="input">

docs/api/com/johnsnowlabs/client/aws/AWSBasicCredentials.html

@@ -3,9 +3,9 @@
         <head>
           <meta http-equiv="X-UA-Compatible" content="IE=edge" />
           <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
-          <title>Spark NLP 6.0.5 ScalaDoc  - com.johnsnowlabs.client.aws.AWSBasicCredentials</title>
-          <meta name="description" content="Spark NLP 6.0.5 ScalaDoc - com.johnsnowlabs.client.aws.AWSBasicCredentials" />
-          <meta name="keywords" content="Spark NLP 6.0.5 ScalaDoc com.johnsnowlabs.client.aws.AWSBasicCredentials" />
+          <title>Spark NLP 6.1.0-rc1 ScalaDoc  - com.johnsnowlabs.client.aws.AWSBasicCredentials</title>
+          <meta name="description" content="Spark NLP 6.1.0 - rc1 ScalaDoc - com.johnsnowlabs.client.aws.AWSBasicCredentials" />
+          <meta name="keywords" content="Spark NLP 6.1.0 rc1 ScalaDoc com.johnsnowlabs.client.aws.AWSBasicCredentials" />
           <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
 
 
@@ -28,7 +28,7 @@
         </head>
         <body>
       <div id="search">
-        <span id="doc-title">Spark NLP 6.0.5 ScalaDoc<span id="doc-version"></span></span>
+        <span id="doc-title">Spark NLP 6.1.0-rc1 ScalaDoc<span id="doc-version"></span></span>
         <span class="close-results"><span class="left">&lt;</span> Back</span>
         <div id="textfilter">
           <span class="input">