Merge pull request #1321 from stvoutsin/issue/downgrade-versions

Downgrade version of Python, Spark & required libs

deployments/common/pip/requirements.txt

@@ -1,20 +1,21 @@
-numpy==1.26.2
-scipy==1.11.4
-matplotlib==3.8.2
-grpcio==1.59.3
+numpy==1.21.0
+scipy==1.7.3
+matplotlib==3.4.2
+grpcio==1.53.0
 jupyter==1.0.0
-Cython==0.29.36
-protobuf==4.25.1
-pandas==2.1.3
-healpy==1.16.6
-astropy==5.3.4
-astroquery==0.4.6
-scikit-learn==1.3.2
-joblib==1.3.2
-hdbscan==0.8.33
-pyvo==1.5.1
-pyarrow==14.0.1
-GaiaXPy==2.1.0
+Cython==0.29.23
+protobuf==3.18.3
+pandas==1.2.4
+healpy==1.14.0
+astropy==4.2.1
+astroquery==0.4.1
+scikit-learn==0.24.2
+joblib==1.2.0
+hdbscan==0.8.31
+pyvo==1.1
+pyarrow==12.0.1
+koalas==1.8.2
+GaiaXPy==1.1.4
 git+https://github.com/wfau/[email protected]
 git+https://github.com/stvoutsin/[email protected]
 dustmaps==1.0.12

deployments/hadoop-yarn/ansible/28-install-zeppelin-requirements.yml

@@ -52,3 +52,30 @@
       dnf:
         name:  'gcc'
         state: present
+
+    # Why do we need this ?
+    - name: "Install Python3-devel package"
+      become: true
+      dnf:
+        name:  'python3-devel'
+        state: present
+
+    # This is already done in 23-install-python
+    - name: "Set Python alternative to Python3"
+      become: true
+      command: update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
+    # This is already done in 23-install-python
+    - name: "Create [/etc/profile.d/python.sh]"
+      become: true
+      blockinfile:
+        dest:  '/etc/profile.d/python.sh'
+        state: present
+        owner: 'root'
+        group: 'root'
+        mode:  'u=rw,g=r,o=r'
+        create: true
+        insertafter: 'EOF'
+        marker: '# {mark} Ansible managed Python'
+        block: |
+          alias python=python3

deployments/hadoop-yarn/ansible/29-install-pip-libs.yml

@@ -51,17 +51,22 @@
         name:  'gcc'
         state: present
 
+    # Why do we need this ?
+    - name: "Install Python3-devel package"
+      become: true
+      dnf:
+        name:  'python3-devel'
+        state: present
+
     - name: "Install required system libraries"
       become: true
       dnf:
         name: git,libtiff-devel,libjpeg-devel,libzip-devel,freetype-devel,lcms2-devel,libwebp-devel,tcl-devel,tk-devel,libffi,libffi-devel
         state: present
 
-# Do not upgrade pip on Fedora: https://github.com/pypa/pip/issues/11631
-#
-#    - name: "Upgrade pip"
-#      become: true
-#      command: "pip install --upgrade pip"
+    - name: "Upgrade pip"
+      become: true
+      command: "pip install --upgrade pip"
 
     - name: Copy pip requirements file into tmp
       become: true

deployments/hadoop-yarn/ansible/config/spark.yml

@@ -21,8 +21,8 @@
 #
 
 # Spark vars
-spname: "spark-3.2.4"
-spfull: "spark-3.2.4-bin-hadoop3.2"
+spname: "spark-3.1.2"
+spfull: "spark-3.1.2-bin-hadoop3.2"
 spbase: "/opt"
 sphome: "/opt/spark"
 sphost: "master01"

deployments/hadoop-yarn/ansible/config/zeppelin-54.86-spark-6.26.43.yml

@@ -25,7 +25,7 @@ all:
     vars:
 
         # VM image
-        baseimage: 'gaia-dmp-fedora-cloud-38-1.6'
+        baseimage: 'Fedora-31-1.9'
 
         # Flavor sizes
         zeppelinflavor: 'gaia.vm.cclake.54vcpu'

deployments/hadoop-yarn/ansible/create-all.yml

@@ -59,7 +59,7 @@
 - import_playbook: 22-config-spark-master.yml
 - import_playbook: 34-setup-shuffler.yml
 
-#- import_playbook: 23-install-python.yml
+- import_playbook: 23-install-python.yml
 - import_playbook: 24-install-pyspark.yml
 
 - import_playbook: 27-install-zeppelin.yml

deployments/zeppelin/test/config/basic.json

@@ -2,43 +2,43 @@
 "notebooks" : [
            {
               "name" : "GaiaDMPSetup",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/GaiaDMP_validation.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/GaiaDMP_validation.json",
               "totaltime" : 50,
               "results" : []
            },
            {
               "name" : "Mean_proper_motions_over_the_sky",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Mean_proper_motions_over_the_sky.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Mean_proper_motions_over_the_sky.json",
               "totaltime" : 120,
               "results" : []
            },
            {
               "name" : "Source_counts_over_the_sky.json",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Source_counts_over_the_sky.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Source_counts_over_the_sky.json",
               "totaltime" : 55,
               "results" : []
            },
            {
               "name" : "Good_astrometric_solutions_via_ML_Random_Forrest_classifier",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Good_astrometric_solutions_via_ML_Random_Forrest_classifier.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Good_astrometric_solutions_via_ML_Random_Forrest_classifier.json",
               "totaltime" : 650,
               "results" : []
            },
            {
               "name" : "Working_with_cross_matched_surveys",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Working_with_cross_matched_surveys.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Working_with_cross_matched_surveys.json",
               "totaltime" : 190,
               "results" : []
            },
            {
               "name" : "Working_with_Gaia_XP_spectra.json",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Working_with_Gaia_XP_spectra.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Working_with_Gaia_XP_spectra.json",
               "totaltime" : 6000,
               "results" : []
            },
            {
               "name" : "Library_Validation.json",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/Library_validation.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/Library_validation.json",
               "totaltime" : 10,
               "results" : []
            }

deployments/zeppelin/test/config/full.json

@@ -2,49 +2,49 @@
 "notebooks" : [
            {
               "name" : "GaiaDMPSetup",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/GaiaDMP_validation.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/GaiaDMP_validation.json",
               "totaltime" : 50,
               "results" : []
            },
            {
               "name" : "Mean_proper_motions_over_the_sky",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Mean_proper_motions_over_the_sky.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Mean_proper_motions_over_the_sky.json",
               "totaltime" : 125,
               "results" : []
            },
            {
               "name" : "Source_counts_over_the_sky.json",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Source_counts_over_the_sky.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Source_counts_over_the_sky.json",
               "totaltime" : 55,
               "results" : []
            },
            {
               "name" : "Good_astrometric_solutions_via_ML_Random_Forrest_classifier",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Good_astrometric_solutions_via_ML_Random_Forrest_classifier.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Good_astrometric_solutions_via_ML_Random_Forrest_classifier.json",
               "totaltime" : 650,
               "results" : []
            },
            {
               "name" : "Working_with_cross_matched_surveys",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Working_with_cross_matched_surveys.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Working_with_cross_matched_surveys.json",
               "totaltime" : 6000,
               "results" : []
            },
            {
               "name" : "Working_with_Gaia_XP_spectra.json",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Working_with_Gaia_XP_spectra.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Working_with_Gaia_XP_spectra.json",
               "totaltime" : 190,
               "results" : []
            },
            {
               "name" : "QC_cuts_dev.json",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/QC_cuts_dev.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/QC_cuts_dev.json",
               "totaltime" : 10100,
               "results" : []
            },
            {
               "name" : "Library_Validation.json",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/Library_validation.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/Library_validation.json",
               "totaltime" : 10,
               "results" : []
            }

deployments/zeppelin/test/config/quick-healthchecker.json

@@ -2,25 +2,25 @@
 "notebooks" : [
            {
               "name" : "GaiaDMPSetup",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/GaiaDMP_validation.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/GaiaDMP_validation.json",
               "totaltime" : 3,
               "results" : []
            },
            {
               "name" : "Mean_proper_motions_over_the_sky",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Mean_proper_motions_over_the_sky.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Mean_proper_motions_over_the_sky.json",
               "totaltime" : 125,
               "results" : []
            },
            {
               "name" : "Source_counts_over_the_sky.json",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Source_counts_over_the_sky.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Source_counts_over_the_sky.json",
               "totaltime" : 55,
               "results" : []
            },
            {
               "name" : "Library_Validation.json",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/Library_validation.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/Library_validation.json",
               "totaltime" : 10,
               "results" : []
            }

deployments/zeppelin/test/config/quick.json

@@ -2,25 +2,25 @@
 "notebooks" : [
            {
               "name" : "GaiaDMPSetup",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/GaiaDMP_validation.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/GaiaDMP_validation.json",
               "totaltime" : 50,
               "results" : []
            },
            {
               "name" : "Mean_proper_motions_over_the_sky",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Mean_proper_motions_over_the_sky.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Mean_proper_motions_over_the_sky.json",
               "totaltime" : 125,
               "results" : []
            },
            {
               "name" : "Source_counts_over_the_sky.json",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/public_examples/Source_counts_over_the_sky.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/public_examples/Source_counts_over_the_sky.json",
               "totaltime" : 55,
               "results" : []
            },
            {
               "name" : "Library_Validation.json",
-              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/a4e05c8b9ebe0372c431322d7d66b969dfabb9e1/notebooks/Library_validation.json",
+              "filepath" : "https://raw.githubusercontent.com/wfau/aglais-testing/2881e513ef47d42a69cfa9dc3dd3a95bbd517113/notebooks/Library_validation.json",
               "totaltime" : 10,
               "results" : []
            }