Revert back to previous Spark / Python / Fedora versions

stvoutsin · Apr 11, 2024 · 3844686 · 3844686
1 parent a1a3d5f
commit 3844686
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 9 deletions.
diff --git a/deployments/hadoop-yarn/ansible/28-install-zeppelin-requirements.yml b/deployments/hadoop-yarn/ansible/28-install-zeppelin-requirements.yml
@@ -52,3 +52,30 @@
       dnf:
         name:  'gcc'
         state: present
+
+    # Why do we need this ?
+    - name: "Install Python3-devel package"
+      become: true
+      dnf:
+        name:  'python3-devel'
+        state: present
+
+    # This is already done in 23-install-python
+    - name: "Set Python alternative to Python3"
+      become: true
+      command: update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
+    # This is already done in 23-install-python
+    - name: "Create [/etc/profile.d/python.sh]"
+      become: true
+      blockinfile:
+        dest:  '/etc/profile.d/python.sh'
+        state: present
+        owner: 'root'
+        group: 'root'
+        mode:  'u=rw,g=r,o=r'
+        create: true
+        insertafter: 'EOF'
+        marker: '# {mark} Ansible managed Python'
+        block: |
+          alias python=python3
diff --git a/deployments/hadoop-yarn/ansible/29-install-pip-libs.yml b/deployments/hadoop-yarn/ansible/29-install-pip-libs.yml
@@ -51,17 +51,22 @@
         name:  'gcc'
         state: present
 
+    # Why do we need this ?
+    - name: "Install Python3-devel package"
+      become: true
+      dnf:
+        name:  'python3-devel'
+        state: present
+
     - name: "Install required system libraries"
       become: true
       dnf:
         name: git,libtiff-devel,libjpeg-devel,libzip-devel,freetype-devel,lcms2-devel,libwebp-devel,tcl-devel,tk-devel,libffi,libffi-devel
         state: present
 
-# Do not upgrade pip on Fedora: https://github.com/pypa/pip/issues/11631
-#
-#    - name: "Upgrade pip"
-#      become: true
-#      command: "pip install --upgrade pip"
+    - name: "Upgrade pip"
+      become: true
+      command: "pip install --upgrade pip"
 
     - name: Copy pip requirements file into tmp
       become: true

diff --git a/deployments/hadoop-yarn/ansible/config/spark.yml b/deployments/hadoop-yarn/ansible/config/spark.yml
@@ -21,8 +21,8 @@
 #
 
 # Spark vars
-spname: "spark-3.2.4"
-spfull: "spark-3.2.4-bin-hadoop3.2"
+spname: "spark-3.1.2"
+spfull: "spark-3.1.2-bin-hadoop3.2"
 spbase: "/opt"
 sphome: "/opt/spark"
 sphost: "master01"

diff --git a/deployments/hadoop-yarn/ansible/config/zeppelin-54.86-spark-6.26.43.yml b/deployments/hadoop-yarn/ansible/config/zeppelin-54.86-spark-6.26.43.yml
@@ -25,7 +25,7 @@ all:
     vars:
 
         # VM image
-        baseimage: 'gaia-dmp-fedora-cloud-38-1.6'
+        baseimage: 'Fedora-31-1.9'
 
         # Flavor sizes
         zeppelinflavor: 'gaia.vm.cclake.54vcpu'

diff --git a/deployments/hadoop-yarn/ansible/create-all.yml b/deployments/hadoop-yarn/ansible/create-all.yml
@@ -59,7 +59,7 @@
 - import_playbook: 22-config-spark-master.yml
 - import_playbook: 34-setup-shuffler.yml
 
-#- import_playbook: 23-install-python.yml
+- import_playbook: 23-install-python.yml
 - import_playbook: 24-install-pyspark.yml
 
 - import_playbook: 27-install-zeppelin.yml