BIGTOP-4151: Add pkgSuffix and parentDir build arguments to Zookeeper…

bigtop-deploy/puppet/modules/hadoop_zookeeper/manifests/init.pp

@@ -17,6 +17,20 @@
   $kerberos_realm = "",
 ) {
 
+  $package_suffix = hiera("bigtop::package_suffix", "")
+  $parent_dir = hiera("bigtop::parent_dir", "")
+  $bigtop_base_version = hiera("bigtop::base_version", "")
+
+  if $package_suffix and $package_suffix != '' and $parent_dir and $parent_dir != '' {
+    $bin_home = "${parent_dir}/${bigtop_base_version}/usr/bin"
+  } else {
+    $bin_home = '/usr/bin'
+  }
+
+  $zookeeper_package = "zookeeper${package_suffix}"
+  $zookeeper_server_package = "zookeeper${package_suffix}-server"
+  $zookeeper_service_name = "zookeeper${package_suffix}-server"
+
   class deploy ($roles) {
     if ("zookeeper-client" in $roles) {
       include hadoop_zookeeper::client
@@ -45,15 +59,17 @@
   ) inherits hadoop_zookeeper {
     include hadoop_zookeeper::common
 
-    package { "zookeeper":
+
+
+    package { $zookeeper_package:
       ensure => latest,
       require => Package["jdk"],
     }
 
     if ($kerberos_realm and $kerberos_realm != "") {
       file { '/etc/zookeeper/conf/client-jaas.conf':
         content => template('hadoop_zookeeper/client-jaas.conf'),
-        require => Package['zookeeper'],
+        require => Package[$zookeeper_package],
       }
     }
   }
@@ -69,51 +85,60 @@
   ) inherits hadoop_zookeeper {
     include hadoop_zookeeper::common
 
-    package { "zookeeper-server":
+
+    package { $zookeeper_server_package:
       ensure => latest,
       require => Package["jdk"],
     }
 
-    service { "zookeeper-server":
+    file { $datadir:
+      ensure => directory,
+      owner  => 'zookeeper',
+      group  => 'zookeeper',
+      mode   => '0755',
+      require => Package[$zookeeper_server_package],
+    }
+
+    service { $zookeeper_service_name:
       ensure => running,
-      require => [ Package["zookeeper-server"],
+      require => [ Package[$zookeeper_server_package],
                    Exec["zookeeper-server-initialize"] ],
       subscribe => [ File["/etc/zookeeper/conf/zoo.cfg"],
-                     File["/var/lib/zookeeper/myid"] ],
+                     File["${datadir}/myid"] ],
       hasrestart => true,
       hasstatus => true,
     }
 
     file { "/etc/zookeeper/conf/zoo.cfg":
       content => template("hadoop_zookeeper/zoo.cfg"),
-      require => Package["zookeeper-server"],
+      require => Package[$zookeeper_server_package],
     }
 
-    file { "/var/lib/zookeeper/myid":
+    file { "${datadir}/myid":
       content => inline_template("<%= @myid %>"),
-      require => Package["zookeeper-server"],
+      require => [Package[$zookeeper_server_package], File[$datadir]],
     }
 
     exec { "zookeeper-server-initialize":
-      command => "/usr/bin/zookeeper-server-initialize",
+      command => "$bin_home/zookeeper-server-initialize",
       user    => "zookeeper",
-      creates => "/var/lib/zookeeper/version-2",
-      require => Package["zookeeper-server"],
+      creates => "${datadir}/version-2",
+      require => [Package[$zookeeper_server_package], File[$datadir]],
     }
 
     if ($kerberos_realm and $kerberos_realm != "") {
       require kerberos::client
 
       kerberos::host_keytab { "zookeeper":
         spnego  => true,
-        require => Package["zookeeper-server"],
-        before  => Service["zookeeper-server"],
+        require => Package[$zookeeper_server_package],
+        before  => Service[$zookeeper_service_name],
       }
 
       file { "/etc/zookeeper/conf/server-jaas.conf":
         content => template("hadoop_zookeeper/server-jaas.conf"),
-        require => Package["zookeeper-server"],
-        notify  => Service["zookeeper-server"],
+        require => Package[$zookeeper_server_package],
+        notify  => Service[$zookeeper_service_name],
       }
     }
   }

bigtop-packages/src/deb/zookeeper/control

@@ -20,27 +20,27 @@ Build-Depends: debhelper (>= 7.0.50~), autoconf, automake
 Standards-Version: 3.9.6
 Homepage: http://zookeeper.apache.org/
 
-Package: zookeeper
+Package: zookeeper-pkgsuffix
 Architecture: all
 Depends: adduser, bigtop-utils (>= 0.7)
 Description: A high-performance coordination service for distributed applications.
  ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services.  All of these kinds of services are used in some form or another by distributed applications. Each time they are implemented there is a lot of work that goes into fixing the bugs and race conditions that are inevitable. Because of the difficulty of implementing these kinds of services, applications initially usually skimp on them ,which make them brittle in the presence of change and difficult to manage. Even when done correctly, different implementations of these services lead to management complexity when the applications are deployed.
 
-Package: zookeeper-server
+Package: zookeeper-pkgsuffix-server
 Architecture: all
-Depends: zookeeper (= ${source:Version})
+Depends: zookeeper-pkgsuffix (= ${source:Version})
 Description: The Hadoop Zookeeper server
  This package starts the zookeeper server on startup
 
-Package: zookeeper-rest
+Package: zookeeper-pkgsuffix-rest
 Architecture: all
-Depends: zookeeper (= ${source:Version}), lsb-base
+Depends: zookeeper-pkgsuffix (= ${source:Version}), lsb-base
 Description: The Zookeeper REST server
  This package starts the zookeeper REST server on startup
 
-Package: zookeeper-native
+Package: zookeeper-pkgsuffix-native
 Architecture: any
-Depends: zookeeper (= ${source:Version}),  ${shlibs:Depends} 
+Depends: zookeeper-pkgsuffix (= ${source:Version}),  ${shlibs:Depends}
 Description: C bindings for ZooKeeper clients
  Provides native libraries and development headers for C / C++ ZooKeeper clients. Consists of both single-threaded and multi-threaded implementations.
 

bigtop-packages/src/deb/zookeeper/rules

@@ -28,24 +28,77 @@ zookeeper_pkg_name=zookeeper
 %:
 	dh $@
 
+parent_dir=${PARENT_DIR}
+pkg_name_suffix=${PKG_NAME_SUFFIX}
+
+zookeeper_name=zookeeper
+doc_zookeeper=${parent_dir}/usr/share/doc/${zookeeper_name}
+usr_lib_zookeeper=${parent_dir}/usr/lib/${zookeeper_name}
+var_lib_zookeeper=${parent_dir}/var/lib/${zookeeper_name}
+bin_dir=${parent_dir}/usr/bin
+man_dir=${parent_dir}/usr/share/man
+etc_zookeeper=${parent_dir}/etc/${zookeeper_name}
+etc_zookeeper_conf_dist=${parent_dir}/etc/${zookeeper_name}/conf.dist
+etc_default=${parent_dir}/etc/default
+etc_default_zookeeper=${etc_default}/${zookeeper_name}
+include_dir=${parent_dir}/usr/include
+lib_dir=${parent_dir}/usr/lib/
+
+define gen_rule
+    $(foreach item,postinst preinst prerm,\
+        if [ -f debian/zookeeper$(1).$(item) ]; then \
+            sed -i -e 's:@etc_zookeeper@:${etc_zookeeper}:g' -e 's:@var_lib_zookeeper@:${var_lib_zookeeper}:g' debian/zookeeper$(1).$(item); \
+            if [ "${pkg_name_suffix}" != "" ] ;then mv debian/zookeeper$(1).$(item) debian/zookeeper${pkg_name_suffix}$(1).$(item);fi \
+        fi; \
+    )
+endef
+
+.PHONY: update_control
+.PHONY: gen_files
+
+update_control:
+	sed -i 's/-pkgsuffix/${pkg_name_suffix}/g' debian/control
+	if [ "${pkg_name_suffix}" != "" ] && [ -e debian/zookeeper-server.init ]; then mv debian/zookeeper-server.init debian/zookeeper${pkg_name_suffix}-server.init; fi
+	sed -i  -e 's:@etc_default_zookeeper:${etc_default_zookeeper}:g' debian/zookeeper${pkg_name_suffix}-server.init
+	sed -i -e 's:@bin_home:${bin_dir}:g' debian/zookeeper${pkg_name_suffix}-server.init
+
+
+gen_files:
+	$(call gen_rule,)
+
+
+override_dh_auto_clean: update_control
+	dh_auto_clean
+
 override_dh_auto_build:
 	bash debian/do-component-build -Divy.home=${HOME}/.ivy2
 	mkdir -p debian/tmp
 	tar cf - --exclude=debian/\* . | (cd debian/tmp && tar xf -)
 
-override_dh_auto_install:
+override_dh_auto_install: gen_files
 	cp debian/zookeeper.1 debian/zoo.cfg debian/zookeeper.default .
 	bash -x debian/install_zookeeper.sh \
-	  --build-dir=build \
-	  --prefix=debian/$(zookeeper_pkg_name)
+		--build-dir=build \
+		--prefix=debian/$(zookeeper_pkg_name)${pkg_name_suffix} \
+		--doc-dir=${doc_zookeeper} \
+		--lib-dir=${usr_lib_zookeeper} \
+		--bin-dir=${bin_dir} \
+		--man-dir=${man_dir} \
+		--conf-dist-dir=${etc_zookeeper_conf_dist} \
+		--etc-default=${etc_default} \
+		--system-include-dir=${include_dir} \
+		--system-lib-dir=${lib_dir}
 	   # Move native files to a dedicated package
-	   mkdir -p debian/zookeeper-native/usr/bin
-	   mkdir -p debian/zookeeper-native/usr/lib
-	   mv debian/zookeeper/usr/bin/cli_* debian/zookeeper/usr/bin/load_gen debian/zookeeper-native/usr/bin/
-	   mv debian/zookeeper/usr/lib/libzookeeper* debian/zookeeper-native/usr/lib/
-	   mv debian/zookeeper/usr/lib/zookeeper-native debian/zookeeper-native/usr/lib/
-	   mv debian/zookeeper/usr/include debian/zookeeper-native/usr/include
+	   mkdir -p debian/zookeeper${pkg_name_suffix}-native/${bin_dir}
+	   mkdir -p debian/zookeeper${pkg_name_suffix}-native/${lib_dir}
+	   mv debian/zookeeper${pkg_name_suffix}/${bin_dir}/cli_* debian/zookeeper${pkg_name_suffix}/${bin_dir}/load_gen debian/zookeeper${pkg_name_suffix}-native/${bin_dir}
+	   mv debian/zookeeper${pkg_name_suffix}/${lib_dir}/libzookeeper* debian/zookeeper${pkg_name_suffix}-native/${lib_dir}
+	   mv debian/zookeeper${pkg_name_suffix}/${lib_dir}/zookeeper-native debian/zookeeper${pkg_name_suffix}-native/${lib_dir}
+	   mv debian/zookeeper${pkg_name_suffix}/${include_dir} debian/zookeeper${pkg_name_suffix}-native/${include_dir}
 	bash debian/init.d.tmpl debian/zookeeper-rest.svc deb debian/zookeeper-rest.init
 
 override_dh_strip:
 	dh_strip --no-automatic-dbgsym
+
+
+override_dh_strip_nondeterminism:

bigtop-packages/src/deb/zookeeper/zookeeper-server.init

@@ -34,10 +34,10 @@ set -e
 # Autodetect JAVA_HOME if not defined
 . /usr/lib/bigtop-utils/bigtop-detect-javahome
 
-. /etc/default/zookeeper
+. @etc_default_zookeeper
 
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-DAEMON_SCRIPT="/usr/bin/zookeeper-server"
+DAEMON_SCRIPT="@bin_home/zookeeper-server"
 
 NAME=zookeeper-server
 DESC="ZooKeeper daemon"

bigtop-packages/src/deb/zookeeper/zookeeper.postinst

@@ -25,7 +25,7 @@ case "$1" in
     configure)
         # Install config alternatives
         chown -R zookeeper:zookeeper /var/log/zookeeper
-        update-alternatives  --install /etc/${zookeeper_pkg_name}/conf ${zookeeper_pkg_name}-conf /etc/${zookeeper_pkg_name}/conf.dist 30
+        update-alternatives  --install /etc/${zookeeper_pkg_name}/conf ${zookeeper_pkg_name}-conf @etc_zookeeper@/conf.dist 30
     ;;
 
     abort-upgrade|abort-remove|abort-deconfigure)

bigtop-packages/src/deb/zookeeper/zookeeper.preinst

@@ -38,13 +38,13 @@ case "$1" in
                 adduser \
                   --system \
                   --ingroup zookeeper \
-                  --home /var/lib/zookeeper \
+                  --home @var_lib_zookeeper@ \
                   --gecos "ZooKeeper User" \
                   --shell /bin/false \
                   zookeeper >/dev/null
         fi
         install -d -m 0755 -o zookeeper -g zookeeper /var/log/zookeeper
-        install -d -m 0755 -o zookeeper -g zookeeper /var/lib/zookeeper
+        install -d -m 0755 -o zookeeper -g zookeeper @var_lib_zookeeper@
     ;;
 
     abort-upgrade)

bigtop-packages/src/deb/zookeeper/zookeeper.prerm

@@ -25,7 +25,7 @@ set -e
 
 case "$1" in
     remove|upgrade|deconfigure)
-      update-alternatives --remove ${zookeeper_pkg_name}-conf /etc/${zookeeper_pkg_name}/conf.dist || :
+      update-alternatives --remove ${zookeeper_pkg_name}-conf @etc_zookeeper@/conf.dist || :
     ;;
 
     failed-upgrade)

bigtop-tests/smoke-tests/zookeeper/TestZookeeper.groovy

@@ -29,7 +29,7 @@ import static org.apache.bigtop.itest.LogErrorsUtils.logError
 
 class TestZookeeper {
   static private Log LOG = LogFactory.getLog(Object.class)
-
+  static final String ZK_HOME = System.getenv("ZOOKEEPER_HOME") ?: "/usr/lib/zookeeper"
   static Shell sh = new Shell("/bin/bash -s")
 
   @BeforeClass
@@ -47,7 +47,7 @@ class TestZookeeper {
     // Basic test to verify that the server is running, and is in a
     // state that we expect.
     LOG.info('Running zkServer.sh status');
-    sh.exec("/usr/lib/zookeeper/bin/zkServer.sh status");
+    sh.exec("${ZK_HOME}/bin/zkServer.sh status");
     logError(sh);
     assertTrue("Failed ...", sh.getRet() == 0);
 

bigtop.bom

@@ -136,6 +136,7 @@ bigtop {
       name    = 'zookeeper'
       pkg     = name
       rpm_pkg_suffix = "_" + bigtop.base_version.replace(".", "_")
+      deb_pkg_suffix = "-" + bigtop.base_version.replace(".", "-")
       version {
         base  = '3.7.2'
         pkg   = base

packages.gradle

@@ -385,7 +385,7 @@ def genTasks = { target ->
     def final FULL_PARENT_DIR = getParentDir(config.bigtop.base_version, "deb")
     def final DEB_PKG_NAME_SUFFIX =  getPkgNameSuffix(config.bigtop.components[target], "deb").pkgNameSuffix
     def final BIGTOP_BASE_VERSION = "${config.bigtop.base_version}"
-    
+
     exec {
       workingDir PKG_OUTPUT_DIR
       commandLine "dpkg-source -x $SRCDEB".split(' ')
@@ -435,6 +435,7 @@ def genTasks = { target ->
       return
     }
 
+
     def final BIGTOP_BASE_VERSION = "${config.bigtop.base_version}"
      def final BIGTOP_BUILD_STAMP = System.getenv('BIGTOP_BUILD_STAMP') ?:
              config.bigtop.components[target].version.release