From c195efeacf8a15cb065fe73e933ae9855409947f Mon Sep 17 00:00:00 2001 From: George Lester Date: Sat, 12 Dec 2015 11:17:38 -0800 Subject: [PATCH 01/30] Starting to add apk support, based on deb --- lib/fpm/package/apk.rb | 629 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 629 insertions(+) create mode 100644 lib/fpm/package/apk.rb diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb new file mode 100644 index 0000000000..3a96547890 --- /dev/null +++ b/lib/fpm/package/apk.rb @@ -0,0 +1,629 @@ +require "erb" +require "fpm/namespace" +require "fpm/package" +require "fpm/errors" +require "fpm/util" +require "backports" +require "fileutils" +require "digest" + +# Support for debian packages (.deb files) +# +# This class supports both input and output of packages. +class FPM::Package::APK< FPM::Package + + # Map of what scripts are named. + SCRIPT_MAP = { + :before_install => "pre-install", + :after_install => "post-install", + :before_remove => "pre-deinstall", + :after_remove => "post-deinstall", + } unless defined?(SCRIPT_MAP) + + # The list of supported compression types. Default is gz (gzip) + COMPRESSION_TYPES = [ "gz" ] + + private + + # Get the name of this package. See also FPM::Package#name + # + # This accessor actually modifies the name if it has some invalid or unwise + # characters. + def name + if @name =~ /[A-Z]/ + logger.warn("apk packages should not have uppercase characters in their names") + @name = @name.downcase + end + + if @name.include?("_") + logger.warn("apk packages should not include underscores") + @name = @name.gsub(/[_]/, "-") + end + + if @name.include?(" ") + logger.warn("apk packages should not contain spaces") + @name = @name.gsub(/[ ]/, "-") + end + + return @name + end # def name + + def prefix + return (attributes[:prefix] or "/") + end # def prefix + + def input(input_path) + extract_info(input_path) + extract_files(input_path) + end # def input + + def extract_info(package) + + logger.error("Extraction is not yet implemented") + end # def extract_info + + # Parse a 'depends' line from a debian control file. + # + # The expected input 'data' should be everything after the 'Depends: ' string + # + # Example: + # + # parse_depends("foo (>= 3), bar (= 5), baz") + def parse_depends(data) + return [] if data.nil? or data.empty? + # parse dependencies. Debian dependencies come in one of two forms: + # * name + # * name (op version) + # They are all on one line, separated by ", " + + dep_re = /^([^ ]+)(?: \(([>=<]+) ([^)]+)\))?$/ + return data.split(/, */).collect do |dep| + m = dep_re.match(dep) + if m + name, op, version = m.captures + # deb uses ">>" and "<<" for greater and less than respectively. + # fpm wants just ">" and "<" + op = "<" if op == "<<" + op = ">" if op == ">>" + # this is the proper form of dependency + "#{name} #{op} #{version}" + else + # Assume normal form dependency, "name op version". + dep + end + end + end # def parse_depends + + def extract_files(package) + + # unpack the data.tar.{gz,bz2,xz} from the deb package into staging_path + safesystem("ar p #{package} data.tar.gz " \ + "| tar gz -xf - -C #{staging_path}") + end # def extract_files + + def output(output_path) + self.provides = self.provides.collect { |p| fix_provides(p) } + output_check(output_path) + # Abort if the target path already exists. + + # create 'debian-binary' file, required to make a valid debian package + File.write(build_path("debian-binary"), "2.0\n") + + # If we are given --deb-shlibs but no --after-install script, we + # should implicitly create a before/after scripts that run ldconfig + if attributes[:deb_shlibs] + if !script?(:after_install) + logger.info("You gave --deb-shlibs but no --after-install, so " \ + "I am adding an after-install script that runs " \ + "ldconfig to update the system library cache") + scripts[:after_install] = template("deb/ldconfig.sh.erb").result(binding) + end + if !script?(:after_remove) + logger.info("You gave --deb-shlibs but no --after-remove, so " \ + "I am adding an after-remove script that runs " \ + "ldconfig to update the system library cache") + scripts[:after_remove] = template("deb/ldconfig.sh.erb").result(binding) + end + end + + attributes.fetch(:deb_systemd_list, []).each do |systemd| + name = File.basename(systemd, ".service") + dest_systemd = staging_path("lib/systemd/system/#{name}.service") + FileUtils.mkdir_p(File.dirname(dest_systemd)) + FileUtils.cp(systemd, dest_systemd) + File.chmod(0644, dest_systemd) + + # set the attribute with the systemd service name + attributes[:deb_systemd] = name + end + + if script?(:before_upgrade) or script?(:after_upgrade) or attributes[:deb_systemd] + puts "Adding action files" + if script?(:before_install) or script?(:before_upgrade) + scripts[:before_install] = template("deb/preinst_upgrade.sh.erb").result(binding) + end + if script?(:before_remove) or attributes[:deb_systemd] + scripts[:before_remove] = template("deb/prerm_upgrade.sh.erb").result(binding) + end + if script?(:after_install) or script?(:after_upgrade) or attributes[:deb_systemd] + scripts[:after_install] = template("deb/postinst_upgrade.sh.erb").result(binding) + end + if script?(:after_remove) + scripts[:after_remove] = template("deb/postrm_upgrade.sh.erb").result(binding) + end + end + + write_control_tarball + + # Tar up the staging_path into data.tar.{compression type} + case self.attributes[:deb_compression] + when "gz", nil + datatar = build_path("data.tar.gz") + compression = "-z" + when "bzip2" + datatar = build_path("data.tar.bz2") + compression = "-j" + when "xz" + datatar = build_path("data.tar.xz") + compression = "-J" + else + raise FPM::InvalidPackageConfiguration, + "Unknown compression type '#{self.attributes[:deb_compression]}'" + end + + # Write the changelog file + dest_changelog = File.join(staging_path, "usr/share/doc/#{name}/changelog.Debian.gz") + FileUtils.mkdir_p(File.dirname(dest_changelog)) + File.new(dest_changelog, "wb", 0644).tap do |changelog| + Zlib::GzipWriter.new(changelog, Zlib::BEST_COMPRESSION).tap do |changelog_gz| + if attributes[:deb_changelog] + logger.info("Writing user-specified changelog", :source => attributes[:deb_changelog]) + File.new(attributes[:deb_changelog]).tap do |fd| + chunk = nil + # Ruby 1.8.7 doesn't have IO#copy_stream + changelog_gz.write(chunk) while chunk = fd.read(16384) + end.close + else + logger.info("Creating boilerplate changelog file") + changelog_gz.write(template("deb/changelog.erb").result(binding)) + end + end.close + end # No need to close, GzipWriter#close will close it. + + attributes.fetch(:deb_init_list, []).each do |init| + name = File.basename(init, ".init") + dest_init = File.join(staging_path, "etc/init.d/#{name}") + FileUtils.mkdir_p(File.dirname(dest_init)) + FileUtils.cp init, dest_init + File.chmod(0755, dest_init) + end + + attributes.fetch(:deb_default_list, []).each do |default| + name = File.basename(default, ".default") + dest_default = File.join(staging_path, "etc/default/#{name}") + FileUtils.mkdir_p(File.dirname(dest_default)) + FileUtils.cp default, dest_default + File.chmod(0644, dest_default) + end + + attributes.fetch(:deb_upstart_list, []).each do |upstart| + name = File.basename(upstart, ".upstart") + dest_upstart = staging_path("etc/init/#{name}.conf") + FileUtils.mkdir_p(File.dirname(dest_upstart)) + FileUtils.cp(upstart, dest_upstart) + File.chmod(0644, dest_upstart) + + # Install an init.d shim that calls upstart + dest_init = staging_path("etc/init.d/#{name}") + FileUtils.mkdir_p(File.dirname(dest_init)) + FileUtils.ln_s("/lib/init/upstart-job", dest_init) + end + + attributes.fetch(:deb_systemd_list, []).each do |systemd| + name = File.basename(systemd, ".service") + dest_systemd = staging_path("lib/systemd/system/#{name}.service") + FileUtils.mkdir_p(File.dirname(dest_systemd)) + FileUtils.cp(systemd, dest_systemd) + File.chmod(0644, dest_systemd) + end + + write_control_tarball + + # Tar up the staging_path into data.tar.{compression type} + case self.attributes[:deb_compression] + when "gz", nil + datatar = build_path("data.tar.gz") + compression = "-z" + when "bzip2" + datatar = build_path("data.tar.bz2") + compression = "-j" + when "xz" + datatar = build_path("data.tar.xz") + compression = "-J" + else + raise FPM::InvalidPackageConfiguration, + "Unknown compression type '#{self.attributes[:deb_compression]}'" + end + + args = [ tar_cmd, "-C", staging_path, compression ] + data_tar_flags + [ "-cf", datatar, "." ] + safesystem(*args) + + # pack up the .deb, which is just an 'ar' archive with 3 files + # the 'debian-binary' file has to be first + File.expand_path(output_path).tap do |output_path| + ::Dir.chdir(build_path) do + safesystem("ar", "-qc", output_path, "debian-binary", "control.tar.gz", datatar) + end + end + end # def output + + def converted_from(origin) + self.dependencies = self.dependencies.collect do |dep| + fix_dependency(dep) + end.flatten + self.provides = self.provides.collect do |provides| + fix_provides(provides) + end.flatten + + if origin == FPM::Package::Deb + changelog_path = staging_path("usr/share/doc/#{name}/changelog.Debian.gz") + if File.exists?(changelog_path) + logger.debug("Found a deb changelog file, using it.", :path => changelog_path) + attributes[:deb_changelog] = build_path("deb_changelog") + File.open(attributes[:deb_changelog], "w") do |deb_changelog| + Zlib::GzipReader.open(changelog_path) do |gz| + IO::copy_stream(gz, deb_changelog) + end + end + File.unlink(changelog_path) + end + end + end # def converted_from + + def debianize_op(op) + # Operators in debian packaging are <<, <=, =, >= and >> + # So any operator like < or > must be replaced + {:< => "<<", :> => ">>"}[op.to_sym] or op + end + + def fix_dependency(dep) + # Deb dependencies are: NAME (OP VERSION), like "zsh (> 3.0)" + # Convert anything that looks like 'NAME OP VERSION' to this format. + if dep =~ /[\(,\|]/ + # Don't "fix" ones that could appear well formed already. + else + # Convert ones that appear to be 'name op version' + name, op, version = dep.split(/ +/) + if !version.nil? + # Convert strings 'foo >= bar' to 'foo (>= bar)' + dep = "#{name} (#{debianize_op(op)} #{version})" + end + end + + name_re = /^[^ \(]+/ + name = dep[name_re] + if name =~ /[A-Z]/ + logger.warn("Downcasing dependency '#{name}' because deb packages " \ + " don't work so good with uppercase names") + dep = dep.gsub(name_re) { |n| n.downcase } + end + + if dep.include?("_") + logger.warn("Replacing dependency underscores with dashes in '#{dep}' because " \ + "debs don't like underscores") + dep = dep.gsub("_", "-") + end + + # Convert gem ~> X.Y.Z to '>= X.Y.Z' and << X.Y+1.0 + if dep =~ /\(~>/ + name, version = dep.gsub(/[()~>]/, "").split(/ +/)[0..1] + nextversion = version.split(".").collect { |v| v.to_i } + l = nextversion.length + nextversion[l-2] += 1 + nextversion[l-1] = 0 + nextversion = nextversion.join(".") + return ["#{name} (>= #{version})", "#{name} (<< #{nextversion})"] + elsif (m = dep.match(/(\S+)\s+\(!= (.+)\)/)) + # Move '!=' dependency specifications into 'Breaks' + self.attributes[:deb_breaks] ||= [] + self.attributes[:deb_breaks] << dep.gsub(/!=/,"=") + return [] + elsif (m = dep.match(/(\S+)\s+\(= (.+)\)/)) and + self.attributes[:deb_ignore_iteration_in_dependencies?] + # Convert 'foo (= x)' to 'foo (>= x)' and 'foo (<< x+1)' + # but only when flag --ignore-iteration-in-dependencies is passed. + name, version = m[1..2] + nextversion = version.split('.').collect { |v| v.to_i } + nextversion[-1] += 1 + nextversion = nextversion.join(".") + return ["#{name} (>= #{version})", "#{name} (<< #{nextversion})"] + elsif (m = dep.match(/(\S+)\s+\(> (.+)\)/)) + # Convert 'foo (> x) to 'foo (>> x)' + name, version = m[1..2] + return ["#{name} (>> #{version})"] + else + # otherwise the dep is probably fine + return dep.rstrip + end + end # def fix_dependency + + def fix_provides(provides) + name_re = /^[^ \(]+/ + name = provides[name_re] + if name =~ /[A-Z]/ + logger.warn("Downcasing provides '#{name}' because deb packages " \ + " don't work so good with uppercase names") + provides = provides.gsub(name_re) { |n| n.downcase } + end + + if provides.include?("_") + logger.warn("Replacing 'provides' underscores with dashes in '#{provides}' because " \ + "debs don't like underscores") + provides = provides.gsub("_", "-") + end + return provides.rstrip + end + + def control_path(path=nil) + @control_path ||= build_path("control") + FileUtils.mkdir(@control_path) if !File.directory?(@control_path) + + if path.nil? + return @control_path + else + return File.join(@control_path, path) + end + end # def control_path + + def write_control_tarball + # Use custom Debian control file when given ... + write_control # write the control file + write_shlibs # write optional shlibs file + write_scripts # write the maintainer scripts + write_conffiles # write the conffiles + write_debconf # write the debconf files + write_meta_files # write additional meta files + write_triggers # write trigger config to 'triggers' file + write_md5sums # write the md5sums file + + # Make the control.tar.gz + build_path("control.tar.gz").tap do |controltar| + logger.info("Creating", :path => controltar, :from => control_path) + + args = [ tar_cmd, "-C", control_path, "-zcf", controltar, + "--owner=0", "--group=0", "--numeric-owner", "." ] + safesystem(*args) + end + + logger.debug("Removing no longer needed control dir", :path => control_path) + ensure + FileUtils.rm_r(control_path) + end # def write_control_tarball + + def write_control + # warn user if epoch is set + logger.warn("epoch in Version is set", :epoch => self.epoch) if self.epoch + + # calculate installed-size if necessary: + if attributes[:deb_installed_size].nil? + logger.info("No deb_installed_size set, calculating now.") + total = 0 + Find.find(staging_path) do |path| + stat = File.lstat(path) + next if stat.directory? + total += stat.size + end + # Per http://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Installed-Size + # "The disk space is given as the integer value of the estimated + # installed size in bytes, divided by 1024 and rounded up." + attributes[:deb_installed_size] = total / 1024 + end + + # Write the control file + control_path("control").tap do |control| + if attributes[:deb_custom_control] + logger.debug("Using '#{attributes[:deb_custom_control]}' template for the control file") + control_data = File.read(attributes[:deb_custom_control]) + else + logger.debug("Using 'deb.erb' template for the control file") + control_data = template("deb.erb").result(binding) + end + + logger.debug("Writing control file", :path => control) + File.write(control, control_data) + File.chmod(0644, control) + edit_file(control) if attributes[:edit?] + end + end # def write_control + + # Write out the maintainer scripts + # + # SCRIPT_MAP is a map from the package ':after_install' to debian + # 'post_install' names + def write_scripts + SCRIPT_MAP.each do |scriptname, filename| + next unless script?(scriptname) + + control_path(filename).tap do |controlscript| + logger.debug("Writing control script", :source => filename, :target => controlscript) + File.write(controlscript, script(scriptname)) + # deb maintainer scripts are required to be executable + File.chmod(0755, controlscript) + end + end + end # def write_scripts + + def write_conffiles + # check for any init scripts or default files + inits = attributes.fetch(:deb_init_list, []) + defaults = attributes.fetch(:deb_default_list, []) + upstarts = attributes.fetch(:deb_upstart_list, []) + return unless (config_files.any? or inits.any? or defaults.any? or upstarts.any?) + + allconfigs = [] + + # expand recursively a given path to be put in allconfigs + def add_path(path, allconfigs) + # Strip leading / + path = path[1..-1] if path[0,1] == "/" + cfg_path = File.expand_path(path, staging_path) + Find.find(cfg_path) do |p| + if File.file?(p) + allconfigs << p.gsub("#{staging_path}/", '') + end + end + end + + # scan all conf file paths for files and add them + config_files.each do |path| + begin + add_path(path, allconfigs) + rescue Errno::ENOENT + raise FPM::InvalidPackageConfiguration, + "Error trying to use '#{path}' as a config file in the package. Does it exist?" + end + end + + # Also add everything in /etc + begin + if !attributes[:deb_no_default_config_files?] + logger.warn("Debian packaging tools generally labels all files in /etc as config files, " \ + "as mandated by policy, so fpm defaults to this behavior for deb packages. " \ + "You can disable this default behavior with --deb-no-default-config-files flag") + add_path("/etc", allconfigs) + end + rescue Errno::ENOENT + end + + if attributes[:deb_auto_config_files?] + inits.each do |init| + name = File.basename(init, ".init") + initscript = "/etc/init.d/#{name}" + logger.debug("Add conf file declaration for init script", :script => initscript) + allconfigs << initscript[1..-1] + end + defaults.each do |default| + name = File.basename(default, ".default") + confdefaults = "/etc/default/#{name}" + logger.debug("Add conf file declaration for defaults", :default => confdefaults) + allconfigs << confdefaults[1..-1] + end + upstarts.each do |upstart| + name = File.basename(upstart, ".upstart") + upstartscript = "etc/init/#{name}.conf" + logger.debug("Add conf file declaration for upstart script", :script => upstartscript) + allconfigs << upstartscript[1..-1] + end + end + + allconfigs.sort!.uniq! + return unless allconfigs.any? + + control_path("conffiles").tap do |conffiles| + File.open(conffiles, "w") do |out| + allconfigs.each do |cf| + # We need to put the leading / back. Stops lintian relative-conffile error. + out.puts("/" + cf) + end + end + File.chmod(0644, conffiles) + end + end # def write_conffiles + + def write_shlibs + return unless attributes[:deb_shlibs] + logger.info("Adding shlibs", :content => attributes[:deb_shlibs]) + File.open(control_path("shlibs"), "w") do |out| + out.write(attributes[:deb_shlibs]) + end + File.chmod(0644, control_path("shlibs")) + end # def write_shlibs + + def write_debconf + if attributes[:deb_config] + FileUtils.cp(attributes[:deb_config], control_path("config")) + File.chmod(0755, control_path("config")) + end + + if attributes[:deb_templates] + FileUtils.cp(attributes[:deb_templates], control_path("templates")) + File.chmod(0755, control_path("templates")) + end + end # def write_debconf + + def write_meta_files + files = attributes[:deb_meta_file] + return unless files + files.each do |fn| + dest = control_path(File.basename(fn)) + FileUtils.cp(fn, dest) + File.chmod(0644, dest) + end + end + + def write_triggers + lines = [['interest', :deb_interest], + ['activate', :deb_activate]].map { |label, attr| + (attributes[attr] || []).map { |e| "#{label} #{e}\n" } + }.flatten.join('') + + if lines.size > 0 + File.open(control_path("triggers"), 'a') do |f| + f.write "\n" if f.size > 0 + f.write lines + end + end + end + + def write_md5sums + md5_sums = {} + + Find.find(staging_path) do |path| + if File.file?(path) && !File.symlink?(path) + md5 = Digest::MD5.file(path).hexdigest + md5_path = path.gsub("#{staging_path}/", "") + md5_sums[md5_path] = md5 + end + end + + if not md5_sums.empty? + File.open(control_path("md5sums"), "w") do |out| + md5_sums.each do |path, md5| + out.puts "#{md5} #{path}" + end + end + File.chmod(0644, control_path("md5sums")) + end + end # def write_md5sums + + def to_s(format=nil) + # Default format if nil + # git_1.7.9.3-1_amd64.deb + return super("NAME_FULLVERSION_ARCH.TYPE") if format.nil? + return super(format) + end # def to_s + + def data_tar_flags + data_tar_flags = [] + if attributes[:deb_use_file_permissions?].nil? + if !attributes[:deb_user].nil? + if attributes[:deb_user] == 'root' + data_tar_flags += [ "--numeric-owner", "--owner", "0" ] + else + data_tar_flags += [ "--owner", attributes[:deb_user] ] + end + end + + if !attributes[:deb_group].nil? + if attributes[:deb_group] == 'root' + data_tar_flags += [ "--numeric-owner", "--group", "0" ] + else + data_tar_flags += [ "--group", attributes[:deb_group] ] + end + end + end + return data_tar_flags + end # def data_tar_flags + + public(:input, :output, :architecture, :name, :prefix, :converted_from, :to_s, :data_tar_flags) +end # class FPM::Target::Deb From 2f91b8a1a8f86c6c63b2e1d365c417c9ae5d9138 Mon Sep 17 00:00:00 2001 From: George Lester Date: Sun, 13 Dec 2015 19:36:03 -0800 Subject: [PATCH 02/30] APK control tar implementation --- lib/fpm/package/apk.rb | 577 ++++------------------------------- templates/apk/post-deinstall | 2 + templates/apk/post-install | 2 + templates/apk/post-upgrade | 2 + templates/apk/pre-deinstall | 2 + templates/apk/pre-install | 2 + templates/apk/pre-upgrade | 2 + 7 files changed, 65 insertions(+), 524 deletions(-) create mode 100644 templates/apk/post-deinstall create mode 100644 templates/apk/post-install create mode 100644 templates/apk/post-upgrade create mode 100644 templates/apk/pre-deinstall create mode 100644 templates/apk/pre-install create mode 100644 templates/apk/pre-upgrade diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 3a96547890..92d3880726 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -62,568 +62,97 @@ def extract_info(package) logger.error("Extraction is not yet implemented") end # def extract_info - # Parse a 'depends' line from a debian control file. - # - # The expected input 'data' should be everything after the 'Depends: ' string - # - # Example: - # - # parse_depends("foo (>= 3), bar (= 5), baz") - def parse_depends(data) - return [] if data.nil? or data.empty? - # parse dependencies. Debian dependencies come in one of two forms: - # * name - # * name (op version) - # They are all on one line, separated by ", " - - dep_re = /^([^ ]+)(?: \(([>=<]+) ([^)]+)\))?$/ - return data.split(/, */).collect do |dep| - m = dep_re.match(dep) - if m - name, op, version = m.captures - # deb uses ">>" and "<<" for greater and less than respectively. - # fpm wants just ">" and "<" - op = "<" if op == "<<" - op = ">" if op == ">>" - # this is the proper form of dependency - "#{name} #{op} #{version}" - else - # Assume normal form dependency, "name op version". - dep - end - end - end # def parse_depends - def extract_files(package) - + # unpack the data.tar.{gz,bz2,xz} from the deb package into staging_path safesystem("ar p #{package} data.tar.gz " \ "| tar gz -xf - -C #{staging_path}") end # def extract_files def output(output_path) - self.provides = self.provides.collect { |p| fix_provides(p) } - output_check(output_path) - # Abort if the target path already exists. - - # create 'debian-binary' file, required to make a valid debian package - File.write(build_path("debian-binary"), "2.0\n") - - # If we are given --deb-shlibs but no --after-install script, we - # should implicitly create a before/after scripts that run ldconfig - if attributes[:deb_shlibs] - if !script?(:after_install) - logger.info("You gave --deb-shlibs but no --after-install, so " \ - "I am adding an after-install script that runs " \ - "ldconfig to update the system library cache") - scripts[:after_install] = template("deb/ldconfig.sh.erb").result(binding) - end - if !script?(:after_remove) - logger.info("You gave --deb-shlibs but no --after-remove, so " \ - "I am adding an after-remove script that runs " \ - "ldconfig to update the system library cache") - scripts[:after_remove] = template("deb/ldconfig.sh.erb").result(binding) - end - end - attributes.fetch(:deb_systemd_list, []).each do |systemd| - name = File.basename(systemd, ".service") - dest_systemd = staging_path("lib/systemd/system/#{name}.service") - FileUtils.mkdir_p(File.dirname(dest_systemd)) - FileUtils.cp(systemd, dest_systemd) - File.chmod(0644, dest_systemd) + output_check(output_path) - # set the attribute with the systemd service name - attributes[:deb_systemd] = name - end + control_path = build_path("control") + controltar_path = build_path("control.tar.gz") - if script?(:before_upgrade) or script?(:after_upgrade) or attributes[:deb_systemd] - puts "Adding action files" - if script?(:before_install) or script?(:before_upgrade) - scripts[:before_install] = template("deb/preinst_upgrade.sh.erb").result(binding) - end - if script?(:before_remove) or attributes[:deb_systemd] - scripts[:before_remove] = template("deb/prerm_upgrade.sh.erb").result(binding) - end - if script?(:after_install) or script?(:after_upgrade) or attributes[:deb_systemd] - scripts[:after_install] = template("deb/postinst_upgrade.sh.erb").result(binding) - end - if script?(:after_remove) - scripts[:after_remove] = template("deb/postrm_upgrade.sh.erb").result(binding) - end - end + FileUtils.mkdir(control_path) - write_control_tarball - - # Tar up the staging_path into data.tar.{compression type} - case self.attributes[:deb_compression] - when "gz", nil - datatar = build_path("data.tar.gz") - compression = "-z" - when "bzip2" - datatar = build_path("data.tar.bz2") - compression = "-j" - when "xz" - datatar = build_path("data.tar.xz") - compression = "-J" - else - raise FPM::InvalidPackageConfiguration, - "Unknown compression type '#{self.attributes[:deb_compression]}'" - end - - # Write the changelog file - dest_changelog = File.join(staging_path, "usr/share/doc/#{name}/changelog.Debian.gz") - FileUtils.mkdir_p(File.dirname(dest_changelog)) - File.new(dest_changelog, "wb", 0644).tap do |changelog| - Zlib::GzipWriter.new(changelog, Zlib::BEST_COMPRESSION).tap do |changelog_gz| - if attributes[:deb_changelog] - logger.info("Writing user-specified changelog", :source => attributes[:deb_changelog]) - File.new(attributes[:deb_changelog]).tap do |fd| - chunk = nil - # Ruby 1.8.7 doesn't have IO#copy_stream - changelog_gz.write(chunk) while chunk = fd.read(16384) - end.close - else - logger.info("Creating boilerplate changelog file") - changelog_gz.write(template("deb/changelog.erb").result(binding)) - end - end.close - end # No need to close, GzipWriter#close will close it. - - attributes.fetch(:deb_init_list, []).each do |init| - name = File.basename(init, ".init") - dest_init = File.join(staging_path, "etc/init.d/#{name}") - FileUtils.mkdir_p(File.dirname(dest_init)) - FileUtils.cp init, dest_init - File.chmod(0755, dest_init) - end + # control tar. + begin - attributes.fetch(:deb_default_list, []).each do |default| - name = File.basename(default, ".default") - dest_default = File.join(staging_path, "etc/default/#{name}") - FileUtils.mkdir_p(File.dirname(dest_default)) - FileUtils.cp default, dest_default - File.chmod(0644, dest_default) - end + write_pkginfo(control_path) - attributes.fetch(:deb_upstart_list, []).each do |upstart| - name = File.basename(upstart, ".upstart") - dest_upstart = staging_path("etc/init/#{name}.conf") - FileUtils.mkdir_p(File.dirname(dest_upstart)) - FileUtils.cp(upstart, dest_upstart) - File.chmod(0644, dest_upstart) - - # Install an init.d shim that calls upstart - dest_init = staging_path("etc/init.d/#{name}") - FileUtils.mkdir_p(File.dirname(dest_init)) - FileUtils.ln_s("/lib/init/upstart-job", dest_init) - end + # scripts + scripts = write_control_scripts(control_path) - attributes.fetch(:deb_systemd_list, []).each do |systemd| - name = File.basename(systemd, ".service") - dest_systemd = staging_path("lib/systemd/system/#{name}.service") - FileUtils.mkdir_p(File.dirname(dest_systemd)) - FileUtils.cp(systemd, dest_systemd) - File.chmod(0644, dest_systemd) + # zip it + compress_control(control_path, controltar_path) + ensure + FileUtils.rm_r(control_path) end - write_control_tarball - - # Tar up the staging_path into data.tar.{compression type} - case self.attributes[:deb_compression] - when "gz", nil - datatar = build_path("data.tar.gz") - compression = "-z" - when "bzip2" - datatar = build_path("data.tar.bz2") - compression = "-j" - when "xz" - datatar = build_path("data.tar.xz") - compression = "-J" - else - raise FPM::InvalidPackageConfiguration, - "Unknown compression type '#{self.attributes[:deb_compression]}'" + # data tar. + begin + ensure end - args = [ tar_cmd, "-C", staging_path, compression ] + data_tar_flags + [ "-cf", datatar, "." ] - safesystem(*args) - - # pack up the .deb, which is just an 'ar' archive with 3 files - # the 'debian-binary' file has to be first - File.expand_path(output_path).tap do |output_path| - ::Dir.chdir(build_path) do - safesystem("ar", "-qc", output_path, "debian-binary", "control.tar.gz", datatar) - end - end - end # def output - - def converted_from(origin) - self.dependencies = self.dependencies.collect do |dep| - fix_dependency(dep) - end.flatten - self.provides = self.provides.collect do |provides| - fix_provides(provides) - end.flatten - - if origin == FPM::Package::Deb - changelog_path = staging_path("usr/share/doc/#{name}/changelog.Debian.gz") - if File.exists?(changelog_path) - logger.debug("Found a deb changelog file, using it.", :path => changelog_path) - attributes[:deb_changelog] = build_path("deb_changelog") - File.open(attributes[:deb_changelog], "w") do |deb_changelog| - Zlib::GzipReader.open(changelog_path) do |gz| - IO::copy_stream(gz, deb_changelog) - end - end - File.unlink(changelog_path) - end + # concatenate the two into a real apk. + begin + ensure end - end # def converted_from - def debianize_op(op) - # Operators in debian packaging are <<, <=, =, >= and >> - # So any operator like < or > must be replaced - {:< => "<<", :> => ">>"}[op.to_sym] or op + logger.warn("apk output to is not implemented") end - def fix_dependency(dep) - # Deb dependencies are: NAME (OP VERSION), like "zsh (> 3.0)" - # Convert anything that looks like 'NAME OP VERSION' to this format. - if dep =~ /[\(,\|]/ - # Don't "fix" ones that could appear well formed already. - else - # Convert ones that appear to be 'name op version' - name, op, version = dep.split(/ +/) - if !version.nil? - # Convert strings 'foo >= bar' to 'foo (>= bar)' - dep = "#{name} (#{debianize_op(op)} #{version})" - end - end + def write_pkginfo(base_path) - name_re = /^[^ \(]+/ - name = dep[name_re] - if name =~ /[A-Z]/ - logger.warn("Downcasing dependency '#{name}' because deb packages " \ - " don't work so good with uppercase names") - dep = dep.gsub(name_re) { |n| n.downcase } - end + path = "#{base_path}/.PKGINFO" - if dep.include?("_") - logger.warn("Replacing dependency underscores with dashes in '#{dep}' because " \ - "debs don't like underscores") - dep = dep.gsub("_", "-") - end + pkginfo_io = StringIO::new + package_version = to_s("FULLVERSION") - # Convert gem ~> X.Y.Z to '>= X.Y.Z' and << X.Y+1.0 - if dep =~ /\(~>/ - name, version = dep.gsub(/[()~>]/, "").split(/ +/)[0..1] - nextversion = version.split(".").collect { |v| v.to_i } - l = nextversion.length - nextversion[l-2] += 1 - nextversion[l-1] = 0 - nextversion = nextversion.join(".") - return ["#{name} (>= #{version})", "#{name} (<< #{nextversion})"] - elsif (m = dep.match(/(\S+)\s+\(!= (.+)\)/)) - # Move '!=' dependency specifications into 'Breaks' - self.attributes[:deb_breaks] ||= [] - self.attributes[:deb_breaks] << dep.gsub(/!=/,"=") - return [] - elsif (m = dep.match(/(\S+)\s+\(= (.+)\)/)) and - self.attributes[:deb_ignore_iteration_in_dependencies?] - # Convert 'foo (= x)' to 'foo (>= x)' and 'foo (<< x+1)' - # but only when flag --ignore-iteration-in-dependencies is passed. - name, version = m[1..2] - nextversion = version.split('.').collect { |v| v.to_i } - nextversion[-1] += 1 - nextversion = nextversion.join(".") - return ["#{name} (>= #{version})", "#{name} (<< #{nextversion})"] - elsif (m = dep.match(/(\S+)\s+\(> (.+)\)/)) - # Convert 'foo (> x) to 'foo (>> x)' - name, version = m[1..2] - return ["#{name} (>> #{version})"] - else - # otherwise the dep is probably fine - return dep.rstrip - end - end # def fix_dependency - - def fix_provides(provides) - name_re = /^[^ \(]+/ - name = provides[name_re] - if name =~ /[A-Z]/ - logger.warn("Downcasing provides '#{name}' because deb packages " \ - " don't work so good with uppercase names") - provides = provides.gsub(name_re) { |n| n.downcase } - end + pkginfo_io << "pkgname = #{@name}\n" + pkginfo_io << "pkgver = #{package_version}\n" + pkginfo_io << "datahash = 123\n" - if provides.include?("_") - logger.warn("Replacing 'provides' underscores with dashes in '#{provides}' because " \ - "debs don't like underscores") - provides = provides.gsub("_", "-") - end - return provides.rstrip + File.write(path, pkginfo_io.string) + `cp #{path} /tmp/.PKGINFO` end - def control_path(path=nil) - @control_path ||= build_path("control") - FileUtils.mkdir(@control_path) if !File.directory?(@control_path) - - if path.nil? - return @control_path - else - return File.join(@control_path, path) - end - end # def control_path - - def write_control_tarball - # Use custom Debian control file when given ... - write_control # write the control file - write_shlibs # write optional shlibs file - write_scripts # write the maintainer scripts - write_conffiles # write the conffiles - write_debconf # write the debconf files - write_meta_files # write additional meta files - write_triggers # write trigger config to 'triggers' file - write_md5sums # write the md5sums file - - # Make the control.tar.gz - build_path("control.tar.gz").tap do |controltar| - logger.info("Creating", :path => controltar, :from => control_path) - - args = [ tar_cmd, "-C", control_path, "-zcf", controltar, - "--owner=0", "--group=0", "--numeric-owner", "." ] - safesystem(*args) - end - - logger.debug("Removing no longer needed control dir", :path => control_path) - ensure - FileUtils.rm_r(control_path) - end # def write_control_tarball - - def write_control - # warn user if epoch is set - logger.warn("epoch in Version is set", :epoch => self.epoch) if self.epoch - - # calculate installed-size if necessary: - if attributes[:deb_installed_size].nil? - logger.info("No deb_installed_size set, calculating now.") - total = 0 - Find.find(staging_path) do |path| - stat = File.lstat(path) - next if stat.directory? - total += stat.size - end - # Per http://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Installed-Size - # "The disk space is given as the integer value of the estimated - # installed size in bytes, divided by 1024 and rounded up." - attributes[:deb_installed_size] = total / 1024 - end - - # Write the control file - control_path("control").tap do |control| - if attributes[:deb_custom_control] - logger.debug("Using '#{attributes[:deb_custom_control]}' template for the control file") - control_data = File.read(attributes[:deb_custom_control]) - else - logger.debug("Using 'deb.erb' template for the control file") - control_data = template("deb.erb").result(binding) - end - - logger.debug("Writing control file", :path => control) - File.write(control, control_data) - File.chmod(0644, control) - edit_file(control) if attributes[:edit?] - end - end # def write_control - - # Write out the maintainer scripts - # - # SCRIPT_MAP is a map from the package ':after_install' to debian - # 'post_install' names - def write_scripts - SCRIPT_MAP.each do |scriptname, filename| - next unless script?(scriptname) - - control_path(filename).tap do |controlscript| - logger.debug("Writing control script", :source => filename, :target => controlscript) - File.write(controlscript, script(scriptname)) - # deb maintainer scripts are required to be executable - File.chmod(0755, controlscript) - end - end - end # def write_scripts - - def write_conffiles - # check for any init scripts or default files - inits = attributes.fetch(:deb_init_list, []) - defaults = attributes.fetch(:deb_default_list, []) - upstarts = attributes.fetch(:deb_upstart_list, []) - return unless (config_files.any? or inits.any? or defaults.any? or upstarts.any?) - - allconfigs = [] - - # expand recursively a given path to be put in allconfigs - def add_path(path, allconfigs) - # Strip leading / - path = path[1..-1] if path[0,1] == "/" - cfg_path = File.expand_path(path, staging_path) - Find.find(cfg_path) do |p| - if File.file?(p) - allconfigs << p.gsub("#{staging_path}/", '') - end - end - end - - # scan all conf file paths for files and add them - config_files.each do |path| - begin - add_path(path, allconfigs) - rescue Errno::ENOENT - raise FPM::InvalidPackageConfiguration, - "Error trying to use '#{path}' as a config file in the package. Does it exist?" - end - end - - # Also add everything in /etc - begin - if !attributes[:deb_no_default_config_files?] - logger.warn("Debian packaging tools generally labels all files in /etc as config files, " \ - "as mandated by policy, so fpm defaults to this behavior for deb packages. " \ - "You can disable this default behavior with --deb-no-default-config-files flag") - add_path("/etc", allconfigs) - end - rescue Errno::ENOENT - end - - if attributes[:deb_auto_config_files?] - inits.each do |init| - name = File.basename(init, ".init") - initscript = "/etc/init.d/#{name}" - logger.debug("Add conf file declaration for init script", :script => initscript) - allconfigs << initscript[1..-1] - end - defaults.each do |default| - name = File.basename(default, ".default") - confdefaults = "/etc/default/#{name}" - logger.debug("Add conf file declaration for defaults", :default => confdefaults) - allconfigs << confdefaults[1..-1] - end - upstarts.each do |upstart| - name = File.basename(upstart, ".upstart") - upstartscript = "etc/init/#{name}.conf" - logger.debug("Add conf file declaration for upstart script", :script => upstartscript) - allconfigs << upstartscript[1..-1] - end - end - - allconfigs.sort!.uniq! - return unless allconfigs.any? - - control_path("conffiles").tap do |conffiles| - File.open(conffiles, "w") do |out| - allconfigs.each do |cf| - # We need to put the leading / back. Stops lintian relative-conffile error. - out.puts("/" + cf) - end - end - File.chmod(0644, conffiles) - end - end # def write_conffiles + # Writes each control script from template into the build path, + # in the folder given by [base_path] + def write_control_scripts(base_path) - def write_shlibs - return unless attributes[:deb_shlibs] - logger.info("Adding shlibs", :content => attributes[:deb_shlibs]) - File.open(control_path("shlibs"), "w") do |out| - out.write(attributes[:deb_shlibs]) - end - File.chmod(0644, control_path("shlibs")) - end # def write_shlibs + scripts = ["pre-install", + "post-install", + "pre-deinstall", + "post-deinstall", + "pre-upgrade", + "post-upgrade" + ] - def write_debconf - if attributes[:deb_config] - FileUtils.cp(attributes[:deb_config], control_path("config")) - File.chmod(0755, control_path("config")) - end + scripts.each do |path| - if attributes[:deb_templates] - FileUtils.cp(attributes[:deb_templates], control_path("templates")) - File.chmod(0755, control_path("templates")) - end - end # def write_debconf - - def write_meta_files - files = attributes[:deb_meta_file] - return unless files - files.each do |fn| - dest = control_path(File.basename(fn)) - FileUtils.cp(fn, dest) - File.chmod(0644, dest) + script_path = "#{base_path}/#{path}" + File.write(script_path, template("apk/#{path}").result(binding)) end end - def write_triggers - lines = [['interest', :deb_interest], - ['activate', :deb_activate]].map { |label, attr| - (attributes[attr] || []).map { |e| "#{label} #{e}\n" } - }.flatten.join('') - - if lines.size > 0 - File.open(control_path("triggers"), 'a') do |f| - f.write "\n" if f.size > 0 - f.write lines - end - end - end + # Compresses the current contents of the given + def compress_control(path, target_path) - def write_md5sums - md5_sums = {} - - Find.find(staging_path) do |path| - if File.file?(path) && !File.symlink?(path) - md5 = Digest::MD5.file(path).hexdigest - md5_path = path.gsub("#{staging_path}/", "") - md5_sums[md5_path] = md5 - end - end + args = [ tar_cmd, "-C", path, "-zcf", target_path, + "--owner=0", "--group=0", "--numeric-owner", "." ] + safesystem(*args) - if not md5_sums.empty? - File.open(control_path("md5sums"), "w") do |out| - md5_sums.each do |path, md5| - out.puts "#{md5} #{path}" - end - end - File.chmod(0644, control_path("md5sums")) - end - end # def write_md5sums + `cp #{target_path} /tmp/control.tar.gz` + end def to_s(format=nil) - # Default format if nil - # git_1.7.9.3-1_amd64.deb return super("NAME_FULLVERSION_ARCH.TYPE") if format.nil? return super(format) - end # def to_s - - def data_tar_flags - data_tar_flags = [] - if attributes[:deb_use_file_permissions?].nil? - if !attributes[:deb_user].nil? - if attributes[:deb_user] == 'root' - data_tar_flags += [ "--numeric-owner", "--owner", "0" ] - else - data_tar_flags += [ "--owner", attributes[:deb_user] ] - end - end - - if !attributes[:deb_group].nil? - if attributes[:deb_group] == 'root' - data_tar_flags += [ "--numeric-owner", "--group", "0" ] - else - data_tar_flags += [ "--group", attributes[:deb_group] ] - end - end - end - return data_tar_flags - end # def data_tar_flags + end - public(:input, :output, :architecture, :name, :prefix, :converted_from, :to_s, :data_tar_flags) -end # class FPM::Target::Deb + public(:input, :output, :architecture, :name, :prefix, :converted_from, :to_s) +end diff --git a/templates/apk/post-deinstall b/templates/apk/post-deinstall new file mode 100644 index 0000000000..039e4d0069 --- /dev/null +++ b/templates/apk/post-deinstall @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 diff --git a/templates/apk/post-install b/templates/apk/post-install new file mode 100644 index 0000000000..039e4d0069 --- /dev/null +++ b/templates/apk/post-install @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 diff --git a/templates/apk/post-upgrade b/templates/apk/post-upgrade new file mode 100644 index 0000000000..039e4d0069 --- /dev/null +++ b/templates/apk/post-upgrade @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 diff --git a/templates/apk/pre-deinstall b/templates/apk/pre-deinstall new file mode 100644 index 0000000000..039e4d0069 --- /dev/null +++ b/templates/apk/pre-deinstall @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 diff --git a/templates/apk/pre-install b/templates/apk/pre-install new file mode 100644 index 0000000000..039e4d0069 --- /dev/null +++ b/templates/apk/pre-install @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 diff --git a/templates/apk/pre-upgrade b/templates/apk/pre-upgrade new file mode 100644 index 0000000000..039e4d0069 --- /dev/null +++ b/templates/apk/pre-upgrade @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 From 1aff3d721d253fbcf169c49f49ad22187e5aa8bd Mon Sep 17 00:00:00 2001 From: George Lester Date: Fri, 18 Dec 2015 23:15:00 -0800 Subject: [PATCH 03/30] Implemented tar cutting and concatenation --- lib/fpm/package/apk.rb | 160 +++++++++++++++++++++++++++++++++-------- 1 file changed, 130 insertions(+), 30 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 92d3880726..3e78e8c164 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -74,35 +74,39 @@ def output(output_path) output_check(output_path) control_path = build_path("control") - controltar_path = build_path("control.tar.gz") + controltar_path = build_path("control.tar") + datatar_path = build_path("data.tar") FileUtils.mkdir(control_path) + # data tar. + tar_path(staging_path(""), datatar_path) + # control tar. begin - write_pkginfo(control_path) - - # scripts - scripts = write_control_scripts(control_path) - - # zip it - compress_control(control_path, controltar_path) + write_control_scripts(control_path) + tar_path(control_path, controltar_path) ensure FileUtils.rm_r(control_path) end - # data tar. - begin - ensure - end - # concatenate the two into a real apk. begin + + # cut end-of-tar record from control tar + cut_tar_record(controltar_path) + + # calculate/rewrite sha1 hashes for data tar + hash_datatar(datatar_path) + + # concatenate the two into the final apk + concat_tars(controltar_path, datatar_path, output_path) ensure + logger.warn("apk output to is not implemented") + `rm -rf /tmp/apkfpm` + `cp -r #{build_path("")} /tmp/apkfpm` end - - logger.warn("apk output to is not implemented") end def write_pkginfo(base_path) @@ -114,23 +118,23 @@ def write_pkginfo(base_path) pkginfo_io << "pkgname = #{@name}\n" pkginfo_io << "pkgver = #{package_version}\n" - pkginfo_io << "datahash = 123\n" File.write(path, pkginfo_io.string) - `cp #{path} /tmp/.PKGINFO` end # Writes each control script from template into the build path, # in the folder given by [base_path] def write_control_scripts(base_path) - scripts = ["pre-install", - "post-install", - "pre-deinstall", - "post-deinstall", - "pre-upgrade", - "post-upgrade" - ] + scripts = + [ + "pre-install", + "post-install", + "pre-deinstall", + "post-deinstall", + "pre-upgrade", + "post-upgrade" + ] scripts.each do |path| @@ -139,14 +143,110 @@ def write_control_scripts(base_path) end end - # Compresses the current contents of the given - def compress_control(path, target_path) + # Removes the end-of-tar records from the given [target_path]. + # End of tar records are two contiguous empty tar records at the end of the file + # Taken together, they comprise 1k of null data. + def cut_tar_record(target_path) - args = [ tar_cmd, "-C", path, "-zcf", target_path, - "--owner=0", "--group=0", "--numeric-owner", "." ] - safesystem(*args) + record_length = 0 + contiguous_records = 0 + current_position = 0 + desired_tar_length = 0 + + # Scan to find the location of the two contiguous null records + open(target_path, "rb") do |file| + + until(contiguous_records == 2) + + # skip to header length + file.read(124) + + ascii_length = file.read(12) + if(file.eof?()) + raise StandardError.new("Invalid tar stream, eof before end-of-tar record") + end + + record_length = ascii_length.to_i(8) + logger.info("Record length: #{ascii_length} (#{record_length}), current position: #{(124 + current_position).to_s(16)}") + + if(record_length == 0) + contiguous_records += 1 + else + # If there was a previous null tar, add its header length too. + if(contiguous_records != 0) + desired_tar_length += 512 + end - `cp #{target_path} /tmp/control.tar.gz` + desired_tar_length += 512 + + # tarballs work in 512-byte blocks, round up to the nearest block. + if(record_length % 512 != 0) + record_length += (512 * (1 - (record_length / 512.0))).round + end + current_position += record_length + + # reset, add length. + contiguous_records = 0 + desired_tar_length += record_length + end + + # finish off the read of the header length + file.read(376) + + # skip content of record + file.read(record_length) + end + end + + # Truncate file + if(desired_tar_length <= 0) + raise StandardError.new("Unable to trim apk control tar") + end + + logger.info("Truncating '#{target_path}' to #{desired_tar_length}") + File.truncate(target_path, desired_tar_length) + end + + # Rewrites the tar file located at the given [target_tar_path] + # to have its file headers use a sha1 checksum. + def hash_datatar(target_tar_path) + + end + + # Concatenates the given [apath] and [bpath] into the given [target_path] + def concat_tars(apath, bpath, target_path) + + target_file = open(target_path, "wb") + + open(apath, "rb") do |file| + until(file.eof?()) + target_file.write(file.read(4096)) + end + end + open(bpath, "rb") do |file| + until(file.eof?()) + target_file.write(file.read(4096)) + end + end + end + + # Tars the current contents of the given [path] to the given [target_path]. + def tar_path(path, target_path) + + args = + [ + tar_cmd, + "-C", + path, + "-cf", + target_path, + "--owner=0", + "--group=0", + "--numeric-owner", + "." + ] + + safesystem(*args) end def to_s(format=nil) From 3e17e10481d10fb0cd236a0fb27ec3b0db5b7063 Mon Sep 17 00:00:00 2001 From: George Lester Date: Fri, 18 Dec 2015 23:16:41 -0800 Subject: [PATCH 04/30] Removed debug lines from previous commit --- lib/fpm/package/apk.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 3e78e8c164..d1e112bb55 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -150,7 +150,6 @@ def cut_tar_record(target_path) record_length = 0 contiguous_records = 0 - current_position = 0 desired_tar_length = 0 # Scan to find the location of the two contiguous null records @@ -167,7 +166,6 @@ def cut_tar_record(target_path) end record_length = ascii_length.to_i(8) - logger.info("Record length: #{ascii_length} (#{record_length}), current position: #{(124 + current_position).to_s(16)}") if(record_length == 0) contiguous_records += 1 @@ -183,7 +181,6 @@ def cut_tar_record(target_path) if(record_length % 512 != 0) record_length += (512 * (1 - (record_length / 512.0))).round end - current_position += record_length # reset, add length. contiguous_records = 0 @@ -203,7 +200,6 @@ def cut_tar_record(target_path) raise StandardError.new("Unable to trim apk control tar") end - logger.info("Truncating '#{target_path}' to #{desired_tar_length}") File.truncate(target_path, desired_tar_length) end From 9d437d38a7f4c3e8c1f17c0a99b4cd7a0fb5ee98 Mon Sep 17 00:00:00 2001 From: George Lester Date: Sat, 19 Dec 2015 17:09:37 -0800 Subject: [PATCH 05/30] Implemented checksum rewriting --- lib/fpm/package/apk.rb | 77 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index d1e112bb55..fd167d5928 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -6,6 +6,7 @@ require "backports" require "fileutils" require "digest" +require 'digest/sha1' # Support for debian packages (.deb files) # @@ -175,16 +176,12 @@ def cut_tar_record(target_path) desired_tar_length += 512 end - desired_tar_length += 512 - # tarballs work in 512-byte blocks, round up to the nearest block. - if(record_length % 512 != 0) - record_length += (512 * (1 - (record_length / 512.0))).round - end + record_length = determine_record_length(record_length) - # reset, add length. + # reset, add length of content and header. contiguous_records = 0 - desired_tar_length += record_length + desired_tar_length += record_length + 512 end # finish off the read of the header length @@ -204,9 +201,62 @@ def cut_tar_record(target_path) end # Rewrites the tar file located at the given [target_tar_path] - # to have its file headers use a sha1 checksum. - def hash_datatar(target_tar_path) + # to have its record headers use a simple checksum, + # and the apk sha1 hash extension. + def hash_datatar(target_path) + + header = "" + data = "" + record_length = 0 + + temporary_file_name = target_path + "~" + + target_file = open(temporary_file_name, "wb") + begin + + success = false + + open(target_path, "rb") do |file| + + until(file.eof?()) + + header = file.read(512) + record_length = header[124..135].to_i(8) + + # blank out header checksum + for i in 148..155 + header[i] = ' ' + end + + # calculate new checksum + checksum = 0 + + for i in 0..511 + checksum += header.getbyte(i) + end + + checksum = checksum.to_s(8).rjust(8, '0') + logger.info("Checksum: #{checksum}") + header[148..155] = checksum + + # write header and data to target file. + target_file.write(header) + + record_length = determine_record_length(record_length) + for i in 0..(record_length / 512) + target_file.write(file.read(512)) + end + end + end + success = true + ensure + target_file.close() + + if(success) + FileUtils.mv(temporary_file_name, target_path) + end + end end # Concatenates the given [apath] and [bpath] into the given [target_path] @@ -226,6 +276,15 @@ def concat_tars(apath, bpath, target_path) end end + # Rounds the given [record_length] to the nearest highest evenly-divisble number of 512. + def determine_record_length(record_length) + + if(record_length % 512 != 0) + record_length += (512 * (1 - (record_length / 512.0))).round + end + return record_length + end + # Tars the current contents of the given [path] to the given [target_path]. def tar_path(path, target_path) From 1a85fa3eb5803bb1f26d311f9fc85438fb526bf6 Mon Sep 17 00:00:00 2001 From: George Lester Date: Sun, 20 Dec 2015 02:13:49 -0800 Subject: [PATCH 06/30] Implemented sha1 hashing and extension records (should work, but apk refuses it) --- lib/fpm/package/apk.rb | 74 ++++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index fd167d5928..3e523af8ca 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -205,9 +205,9 @@ def cut_tar_record(target_path) # and the apk sha1 hash extension. def hash_datatar(target_path) - header = "" - data = "" - record_length = 0 + header = extension_header = "" + data = extension_data = "" + record_length = extension_length = 0 temporary_file_name = target_path + "~" @@ -223,29 +223,30 @@ def hash_datatar(target_path) header = file.read(512) record_length = header[124..135].to_i(8) - # blank out header checksum - for i in 148..155 - header[i] = ' ' - end + data = "" + record_length = determine_record_length(record_length) - # calculate new checksum - checksum = 0 + for i in 0..(record_length / 512) + data += file.read(512) + end + extension_header = "" for i in 0..511 - checksum += header.getbyte(i) + extension_header += '\0' end - checksum = checksum.to_s(8).rjust(8, '0') - logger.info("Checksum: #{checksum}") - header[148..155] = checksum + # hash data contents with sha1 + extension_data = hash_record(data) + extension_header[124..135] = extension_data.length.to_s(8).rjust(12, '0') + extension_header = checksum_header(extension_header) + + # write extension record + target_file.write(extension_header) + target_file.write(extension_data) # write header and data to target file. target_file.write(header) - - record_length = determine_record_length(record_length) - for i in 0..(record_length / 512) - target_file.write(file.read(512)) - end + target_file.write(data) end end @@ -285,6 +286,43 @@ def determine_record_length(record_length) return record_length end + # Checksums the entire contents of the given [header] + # Writes the resultant checksum into indices 148-155 of the same [header], + # and returns the modified header. + # 148-155 is the "size" range in a tar/ustar header. + def checksum_header(header) + + # blank out header checksum + for i in 148..155 + header[i] = ' ' + end + + # calculate new checksum + checksum = 0 + + for i in 0..511 + checksum += header.getbyte(i) + end + + checksum = checksum.to_s(8).rjust(8, '0') + header[148..155] = checksum + return header + end + + # SHA-1 hashes the given data, then places it in the APK hash string format + # and returns + def hash_record(data) + + # %u %s=%s\n + # len name=hash + + hash = Digest::SHA1.hexdigest(data) + hash_length = (hash.length * 2) + 512 + 4 # 512 is header length, 4 is magic. + name = "APK-TOOLS.checksum.sha1" + + return "#{hash_length} #{name}=#{hash}" + end + # Tars the current contents of the given [path] to the given [target_path]. def tar_path(path, target_path) From 0b2ea18fffd985537a74b1beacd461fe95de91df Mon Sep 17 00:00:00 2001 From: George Lester Date: Sun, 20 Dec 2015 02:22:49 -0800 Subject: [PATCH 07/30] Ensured hashed extension data is padded to nearest 512 byte chunk. --- lib/fpm/package/apk.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 3e523af8ca..414a6df976 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -320,7 +320,13 @@ def hash_record(data) hash_length = (hash.length * 2) + 512 + 4 # 512 is header length, 4 is magic. name = "APK-TOOLS.checksum.sha1" - return "#{hash_length} #{name}=#{hash}" + ret = "#{hash_length} #{name}=#{hash}" + + # pad out the result + until(ret.length % 512 == 0) + ret += '\0' + end + return ret end # Tars the current contents of the given [path] to the given [target_path]. From 81b9ff6a9cdf951a10e535472f1742239ceccffd Mon Sep 17 00:00:00 2001 From: George Lester Date: Sun, 20 Dec 2015 02:35:30 -0800 Subject: [PATCH 08/30] Fixed accidental literal '\0' instead of null terminators --- lib/fpm/package/apk.rb | 71 ++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 414a6df976..831046d472 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -212,46 +212,45 @@ def hash_datatar(target_path) temporary_file_name = target_path + "~" target_file = open(temporary_file_name, "wb") + file = open(target_path, "rb") begin success = false + until(file.eof?()) - open(target_path, "rb") do |file| - - until(file.eof?()) + header = file.read(512) + record_length = header[124..135].to_i(8) - header = file.read(512) - record_length = header[124..135].to_i(8) + data = "" + record_length = determine_record_length(record_length) - data = "" - record_length = determine_record_length(record_length) + for i in 0..(record_length / 512) + data += file.read(512) + end - for i in 0..(record_length / 512) - data += file.read(512) - end + extension_header = "" + for i in 0..511 + extension_header << "\0" + end - extension_header = "" - for i in 0..511 - extension_header += '\0' - end + # hash data contents with sha1 + extension_data = hash_record(data) + extension_header[124..135] = extension_data.length.to_s(8).rjust(12, '0') + extension_header = checksum_header(extension_header) - # hash data contents with sha1 - extension_data = hash_record(data) - extension_header[124..135] = extension_data.length.to_s(8).rjust(12, '0') - extension_header = checksum_header(extension_header) + logger.info("Extension header: #{extension_header}") - # write extension record - target_file.write(extension_header) - target_file.write(extension_data) + # write extension record + target_file.write(extension_header) + target_file.write(extension_data) - # write header and data to target file. - target_file.write(header) - target_file.write(data) - end + # write header and data to target file. + target_file.write(header) + target_file.write(data) end - success = true ensure + file.close() target_file.close() if(success) @@ -265,15 +264,19 @@ def concat_tars(apath, bpath, target_path) target_file = open(target_path, "wb") - open(apath, "rb") do |file| - until(file.eof?()) - target_file.write(file.read(4096)) + begin + open(apath, "rb") do |file| + until(file.eof?()) + target_file.write(file.read(4096)) + end end - end - open(bpath, "rb") do |file| - until(file.eof?()) - target_file.write(file.read(4096)) + open(bpath, "rb") do |file| + until(file.eof?()) + target_file.write(file.read(4096)) + end end + ensure + target_file.close() end end @@ -324,7 +327,7 @@ def hash_record(data) # pad out the result until(ret.length % 512 == 0) - ret += '\0' + ret << "\0" end return ret end From c0afd61f56a794aa4fdb332f49468e2a4d95ea89 Mon Sep 17 00:00:00 2001 From: George Lester Date: Sun, 20 Dec 2015 11:04:51 -0800 Subject: [PATCH 09/30] Fixed a menagerie of errors around extension header, and ensured that output concatted tar is also gzipped --- lib/fpm/package/apk.rb | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 831046d472..3df8379f59 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -224,22 +224,17 @@ def hash_datatar(target_path) data = "" record_length = determine_record_length(record_length) - for i in 0..(record_length / 512) + until(data.length == record_length) data += file.read(512) end - extension_header = "" - for i in 0..511 - extension_header << "\0" - end + extension_header = header # hash data contents with sha1 extension_data = hash_record(data) extension_header[124..135] = extension_data.length.to_s(8).rjust(12, '0') extension_header = checksum_header(extension_header) - logger.info("Extension header: #{extension_header}") - # write extension record target_file.write(extension_header) target_file.write(extension_data) @@ -262,21 +257,20 @@ def hash_datatar(target_path) # Concatenates the given [apath] and [bpath] into the given [target_path] def concat_tars(apath, bpath, target_path) - target_file = open(target_path, "wb") - + zip_writer = Zlib::GzipWriter.open(target_path, Zlib::BEST_COMPRESSION) begin open(apath, "rb") do |file| until(file.eof?()) - target_file.write(file.read(4096)) + zip_writer.write(file.read(4096)) end end open(bpath, "rb") do |file| until(file.eof?()) - target_file.write(file.read(4096)) + zip_writer.write(file.read(4096)) end end ensure - target_file.close() + zip_writer.close() end end @@ -320,10 +314,9 @@ def hash_record(data) # len name=hash hash = Digest::SHA1.hexdigest(data) - hash_length = (hash.length * 2) + 512 + 4 # 512 is header length, 4 is magic. - name = "APK-TOOLS.checksum.sha1" + name = "APK-TOOLS.checksum.SHA1" - ret = "#{hash_length} #{name}=#{hash}" + ret = "#{hash.length} #{name}=#{hash}" # pad out the result until(ret.length % 512 == 0) From 52e2c6135948ed74e5a6d009373a2d431772b7b2 Mon Sep 17 00:00:00 2001 From: George Lester Date: Sun, 20 Dec 2015 12:14:05 -0800 Subject: [PATCH 10/30] Fixed extension hash line length --- lib/fpm/package/apk.rb | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 3df8379f59..d914935621 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -307,7 +307,7 @@ def checksum_header(header) end # SHA-1 hashes the given data, then places it in the APK hash string format - # and returns + # then returns. def hash_record(data) # %u %s=%s\n @@ -316,7 +316,21 @@ def hash_record(data) hash = Digest::SHA1.hexdigest(data) name = "APK-TOOLS.checksum.SHA1" - ret = "#{hash.length} #{name}=#{hash}" + ret = "#{name}=#{hash}\n" + + # the length requirement needs to know its own length too, because the length + # is the entire length of the line, not just the contents. + length = ret.length + line_length = length.to_s + length += line_length.length + candidate_ret = "#{line_length} #{ret}" + + if(candidate_ret.length != length) + length += 1 + candidate_ret = "#{length.to_s} #{ret}" + end + + ret = candidate_ret # pad out the result until(ret.length % 512 == 0) From 6078704f9a458056c19028738957db9c4ce10ef2 Mon Sep 17 00:00:00 2001 From: George Lester Date: Sun, 20 Dec 2015 12:45:19 -0800 Subject: [PATCH 11/30] Revised the method by which chunks are rounded to the nearest 512 --- lib/fpm/package/apk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index d914935621..3792f289c1 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -278,7 +278,7 @@ def concat_tars(apath, bpath, target_path) def determine_record_length(record_length) if(record_length % 512 != 0) - record_length += (512 * (1 - (record_length / 512.0))).round + record_length = (record_length + 511) & ~511; end return record_length end From 9a0ce2304cd569a35521555fa3523dc877abe3b8 Mon Sep 17 00:00:00 2001 From: George Lester Date: Sun, 20 Dec 2015 12:48:24 -0800 Subject: [PATCH 12/30] Ensured that extension headers are only added to file contents, not directories or nul records --- lib/fpm/package/apk.rb | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 3792f289c1..631c1b0113 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -219,6 +219,7 @@ def hash_datatar(target_path) until(file.eof?()) header = file.read(512) + typeflag = header[156] record_length = header[124..135].to_i(8) data = "" @@ -228,16 +229,19 @@ def hash_datatar(target_path) data += file.read(512) end - extension_header = header + # If it's not a null record, and not a directory, do extension hash. + if(typeflag != '' && typeflag != '5') + extension_header = header - # hash data contents with sha1 - extension_data = hash_record(data) - extension_header[124..135] = extension_data.length.to_s(8).rjust(12, '0') - extension_header = checksum_header(extension_header) + # hash data contents with sha1 + extension_data = hash_record(data) + extension_header[124..135] = extension_data.length.to_s(8).rjust(12, '0') + extension_header = checksum_header(extension_header) - # write extension record - target_file.write(extension_header) - target_file.write(extension_data) + # write extension record + target_file.write(extension_header) + target_file.write(extension_data) + end # write header and data to target file. target_file.write(header) From 0dcf15ab8984ce691be9688ab90b97b882033820 Mon Sep 17 00:00:00 2001 From: George Lester Date: Wed, 23 Dec 2015 17:24:54 -0800 Subject: [PATCH 13/30] Fixed hanging nul-records on data tar. Ensured both tars are -individually- zipped before concatenation. --- lib/fpm/package/apk.rb | 50 ++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 631c1b0113..62c619dbef 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -102,7 +102,7 @@ def output(output_path) hash_datatar(datatar_path) # concatenate the two into the final apk - concat_tars(controltar_path, datatar_path, output_path) + concat_zip_tars(controltar_path, datatar_path, output_path) ensure logger.warn("apk output to is not implemented") `rm -rf /tmp/apkfpm` @@ -205,6 +205,7 @@ def cut_tar_record(target_path) # and the apk sha1 hash extension. def hash_datatar(target_path) + empty_records = 0 header = extension_header = "" data = extension_data = "" record_length = extension_length = 0 @@ -216,12 +217,13 @@ def hash_datatar(target_path) begin success = false - until(file.eof?()) + until(file.eof?() || empty_records == 2) header = file.read(512) typeflag = header[156] record_length = header[124..135].to_i(8) + logger.info("Read chunk, typeflag: '#{typeflag}', length: #{record_length}") data = "" record_length = determine_record_length(record_length) @@ -229,12 +231,19 @@ def hash_datatar(target_path) data += file.read(512) end + # empty record, check to see if we're EOF. + if(typeflag == "\0") + empty_records += 1 + loop + end + # If it's not a null record, and not a directory, do extension hash. - if(typeflag != '' && typeflag != '5') - extension_header = header + if(typeflag != "\0" && typeflag != '5') + extension_header = header.dup() # hash data contents with sha1 extension_data = hash_record(data) + extension_header[156] = 'x' extension_header[124..135] = extension_data.length.to_s(8).rjust(12, '0') extension_header = checksum_header(extension_header) @@ -258,23 +267,40 @@ def hash_datatar(target_path) end end - # Concatenates the given [apath] and [bpath] into the given [target_path] - def concat_tars(apath, bpath, target_path) + # Concatenates each of the given [apath] and [bpath] into the given [target_path] + def concat_zip_tars(apath, bpath, target_path) - zip_writer = Zlib::GzipWriter.open(target_path, Zlib::BEST_COMPRESSION) - begin + temp_apath = apath + "~" + temp_bpath = bpath + "~" + + Zlib::GzipWriter.open(temp_apath, Zlib::BEST_COMPRESSION) do |target_writer| open(apath, "rb") do |file| until(file.eof?()) - zip_writer.write(file.read(4096)) + target_writer.write(file.read(4096)) end end + end + + Zlib::GzipWriter.open(temp_bpath, Zlib::BEST_COMPRESSION) do |target_writer| open(bpath, "rb") do |file| until(file.eof?()) - zip_writer.write(file.read(4096)) + target_writer.write(file.read(4096)) + end + end + end + + # concat both into one. + File.open(target_path, "wb") do |target_writer| + open(temp_apath, "rb") do |file| + until(file.eof?()) + target_writer.write(file.read(4096)) + end + end + open(temp_bpath, "rb") do |file| + until(file.eof?()) + target_writer.write(file.read(4096)) end end - ensure - zip_writer.close() end end From a46aaac2e84a20425bed0a85564da45d478523c1 Mon Sep 17 00:00:00 2001 From: George Lester Date: Wed, 23 Dec 2015 20:32:41 -0800 Subject: [PATCH 14/30] Removed debug line from last commit --- lib/fpm/package/apk.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 62c619dbef..8a57bbffd6 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -223,7 +223,6 @@ def hash_datatar(target_path) typeflag = header[156] record_length = header[124..135].to_i(8) - logger.info("Read chunk, typeflag: '#{typeflag}', length: #{record_length}") data = "" record_length = determine_record_length(record_length) From 5ef04f0440c25b3a2fae85253f5c568a8b2a6853 Mon Sep 17 00:00:00 2001 From: George Lester Date: Thu, 24 Dec 2015 16:01:12 -0800 Subject: [PATCH 15/30] First working apk package that installs with no errors --- lib/fpm/package/apk.rb | 61 +++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 8a57bbffd6..e1ebb22ad2 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -139,7 +139,7 @@ def write_control_scripts(base_path) scripts.each do |path| - script_path = "#{base_path}/#{path}" + script_path = "#{base_path}/.#{path}" File.write(script_path, template("apk/#{path}").result(binding)) end end @@ -205,7 +205,6 @@ def cut_tar_record(target_path) # and the apk sha1 hash extension. def hash_datatar(target_path) - empty_records = 0 header = extension_header = "" data = extension_data = "" record_length = extension_length = 0 @@ -217,7 +216,7 @@ def hash_datatar(target_path) begin success = false - until(file.eof?() || empty_records == 2) + until(file.eof?()) header = file.read(512) typeflag = header[156] @@ -230,18 +229,17 @@ def hash_datatar(target_path) data += file.read(512) end - # empty record, check to see if we're EOF. - if(typeflag == "\0") - empty_records += 1 - loop - end - - # If it's not a null record, and not a directory, do extension hash. - if(typeflag != "\0" && typeflag != '5') + # If it's not a null record, do extension hash. + if(typeflag != "\0") extension_header = header.dup() - # hash data contents with sha1 - extension_data = hash_record(data) + # hash data contents with sha1, if there is any content. + if(typeflag == '5') + extension_data = hash_record(data) + else + extension_data = "" + end + extension_header[156] = 'x' extension_header[124..135] = extension_data.length.to_s(8).rjust(12, '0') extension_header = checksum_header(extension_header) @@ -272,7 +270,7 @@ def concat_zip_tars(apath, bpath, target_path) temp_apath = apath + "~" temp_bpath = bpath + "~" - Zlib::GzipWriter.open(temp_apath, Zlib::BEST_COMPRESSION) do |target_writer| + Zlib::GzipWriter.open(temp_apath) do |target_writer| open(apath, "rb") do |file| until(file.eof?()) target_writer.write(file.read(4096)) @@ -280,7 +278,7 @@ def concat_zip_tars(apath, bpath, target_path) end end - Zlib::GzipWriter.open(temp_bpath, Zlib::BEST_COMPRESSION) do |target_writer| + Zlib::GzipWriter.open(temp_bpath) do |target_writer| open(bpath, "rb") do |file| until(file.eof?()) target_writer.write(file.read(4096)) @@ -371,20 +369,27 @@ def hash_record(data) # Tars the current contents of the given [path] to the given [target_path]. def tar_path(path, target_path) - args = - [ - tar_cmd, - "-C", - path, - "-cf", - target_path, - "--owner=0", - "--group=0", - "--numeric-owner", - "." - ] + File::Dir::chdir(path) do + entries = File::Dir::glob("**", File::FNM_DOTMATCH) + + args = + [ + tar_cmd, + "-f", + target_path, + "-c" + ] + + entries.each do |entry| + unless(entry == '..' || entry == '.') + args = args << entry + logger.info("Adding #{entry}") + end + end - safesystem(*args) + logger.info(args) + safesystem(*args) + end end def to_s(format=nil) From feb16beaa6ab4d4c06a3e46b3b7dee86996d7076 Mon Sep 17 00:00:00 2001 From: George Lester Date: Thu, 24 Dec 2015 17:13:05 -0800 Subject: [PATCH 16/30] Cleaned up some logic, and ensured that only two nul records are ever written to the resultant tar. --- lib/fpm/package/apk.rb | 44 ++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index e1ebb22ad2..609dbb2d6a 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -13,6 +13,12 @@ # This class supports both input and output of packages. class FPM::Package::APK< FPM::Package + TAR_CHUNK_SIZE = 512 + TAR_TYPEFLAG_OFFSET = 156 + TAR_LENGTH_OFFSET_START = 124 + TAR_LENGTH_OFFSET_END = 135 + + # Map of what scripts are named. SCRIPT_MAP = { :before_install => "pre-install", @@ -173,7 +179,7 @@ def cut_tar_record(target_path) else # If there was a previous null tar, add its header length too. if(contiguous_records != 0) - desired_tar_length += 512 + desired_tar_length += TAR_CHUNK_SIZE end # tarballs work in 512-byte blocks, round up to the nearest block. @@ -181,7 +187,7 @@ def cut_tar_record(target_path) # reset, add length of content and header. contiguous_records = 0 - desired_tar_length += record_length + 512 + desired_tar_length += record_length + TAR_CHUNK_SIZE end # finish off the read of the header length @@ -208,6 +214,7 @@ def hash_datatar(target_path) header = extension_header = "" data = extension_data = "" record_length = extension_length = 0 + empty_records = 0 temporary_file_name = target_path + "~" @@ -215,18 +222,17 @@ def hash_datatar(target_path) file = open(target_path, "rb") begin - success = false - until(file.eof?()) + until(file.eof?() || empty_records == 2) - header = file.read(512) - typeflag = header[156] - record_length = header[124..135].to_i(8) + header = file.read(TAR_CHUNK_SIZE) + typeflag = header[TAR_TYPEFLAG_OFFSET] + record_length = header[TAR_LENGTH_OFFSET_START..TAR_LENGTH_OFFSET_END].to_i(8) data = "" record_length = determine_record_length(record_length) until(data.length == record_length) - data += file.read(512) + data += file.read(TAR_CHUNK_SIZE) end # If it's not a null record, do extension hash. @@ -235,32 +241,30 @@ def hash_datatar(target_path) # hash data contents with sha1, if there is any content. if(typeflag == '5') - extension_data = hash_record(data) - else extension_data = "" + else + extension_data = hash_record(data) end - extension_header[156] = 'x' - extension_header[124..135] = extension_data.length.to_s(8).rjust(12, '0') + extension_header[TAR_TYPEFLAG_OFFSET] = 'x' + extension_header[TAR_LENGTH_OFFSET_START..TAR_LENGTH_OFFSET_END] = extension_data.length.to_s(8).rjust(12, '0') extension_header = checksum_header(extension_header) # write extension record target_file.write(extension_header) target_file.write(extension_data) + else + empty_records += 1 end # write header and data to target file. target_file.write(header) target_file.write(data) end - success = true + FileUtils.mv(temporary_file_name, target_path) ensure file.close() target_file.close() - - if(success) - FileUtils.mv(temporary_file_name, target_path) - end end end @@ -270,6 +274,7 @@ def concat_zip_tars(apath, bpath, target_path) temp_apath = apath + "~" temp_bpath = bpath + "~" + # zip each path separately Zlib::GzipWriter.open(temp_apath) do |target_writer| open(apath, "rb") do |file| until(file.eof?()) @@ -369,6 +374,9 @@ def hash_record(data) # Tars the current contents of the given [path] to the given [target_path]. def tar_path(path, target_path) + # Change directory to the source path, and glob files + # This is done so that we end up with a "flat" archive, that doesn't + # have any path artifacts from the packager's absolute path. File::Dir::chdir(path) do entries = File::Dir::glob("**", File::FNM_DOTMATCH) @@ -383,11 +391,9 @@ def tar_path(path, target_path) entries.each do |entry| unless(entry == '..' || entry == '.') args = args << entry - logger.info("Adding #{entry}") end end - logger.info(args) safesystem(*args) end end From fdf7ccc6e4ffbc04bbe1d2ce46b3c335cae86b14 Mon Sep 17 00:00:00 2001 From: George Lester Date: Thu, 24 Dec 2015 18:00:25 -0800 Subject: [PATCH 17/30] Added magic pax string, and trimmed trailing slash, from tar record names --- lib/fpm/package/apk.rb | 47 +++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 609dbb2d6a..7b468b8109 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -17,7 +17,8 @@ class FPM::Package::APK< FPM::Package TAR_TYPEFLAG_OFFSET = 156 TAR_LENGTH_OFFSET_START = 124 TAR_LENGTH_OFFSET_END = 135 - + TAR_CHECKSUM_OFFSET_START = 148 + TAR_CHECKSUM_OFFSET_END = 155 # Map of what scripts are named. SCRIPT_MAP = { @@ -239,13 +240,26 @@ def hash_datatar(target_path) if(typeflag != "\0") extension_header = header.dup() + # directories have a magic string inserted into their name + full_record_path = extension_header[0..(extension_header.index("\0"))] + full_record_path = add_paxstring(full_record_path) + # hash data contents with sha1, if there is any content. if(typeflag == '5') + extension_data = "" + + # ensure it doesn't end with a slash + if(full_record_path[full_record_path.length-1] == '/') + full_record_path = full_record_path[0..full_record_path.length-1] + end else extension_data = hash_record(data) end + full_record_path = pad_string_to(full_record_path, 100) + extension_header[0..99] = full_record_path + extension_header[TAR_TYPEFLAG_OFFSET] = 'x' extension_header[TAR_LENGTH_OFFSET_START..TAR_LENGTH_OFFSET_END] = extension_data.length.to_s(8).rjust(12, '0') extension_header = checksum_header(extension_header) @@ -322,19 +336,19 @@ def determine_record_length(record_length) def checksum_header(header) # blank out header checksum - for i in 148..155 + for i in TAR_CHECKSUM_OFFSET_START..TAR_CHECKSUM_OFFSET_END header[i] = ' ' end # calculate new checksum checksum = 0 - for i in 0..511 + for i in 0..(TAR_CHUNK_SIZE-1) checksum += header.getbyte(i) end checksum = checksum.to_s(8).rjust(8, '0') - header[148..155] = checksum + header[TAR_CHECKSUM_OFFSET_START..TAR_CHECKSUM_OFFSET_END] = checksum return header end @@ -365,9 +379,7 @@ def hash_record(data) ret = candidate_ret # pad out the result - until(ret.length % 512 == 0) - ret << "\0" - end + ret = pad_string_to(ret, TAR_CHUNK_SIZE) return ret end @@ -398,6 +410,27 @@ def tar_path(path, target_path) end end + # APK adds a "PAX" magic string into most directory names. + # This takes an unchanged directory name and "paxifies" it. + def add_paxstring(ret) + + pax_slash = ret.rindex('/', ret.rindex('/')-1) + if(!pax_slash || pax_slash < 0) + pax_slash = 0 + end + return ret.insert(pax_slash, "/PaxHeaders.14670") + end + + # Appends null zeroes to the end of [ret] until it is divisible by [length]. + # Returns the padded result. + def pad_string_to(ret, length) + + until(ret.length % length == 0) + ret << "\0" + end + return ret + end + def to_s(format=nil) return super("NAME_FULLVERSION_ARCH.TYPE") if format.nil? return super(format) From 55e2e239dabf5b82f6292e448d912d460fd1c126 Mon Sep 17 00:00:00 2001 From: George Lester Date: Thu, 24 Dec 2015 18:04:59 -0800 Subject: [PATCH 18/30] Added checksum spacing to match existing apk implementation --- lib/fpm/package/apk.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 7b468b8109..0e0f4f6594 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -347,8 +347,9 @@ def checksum_header(header) checksum += header.getbyte(i) end - checksum = checksum.to_s(8).rjust(8, '0') - header[TAR_CHECKSUM_OFFSET_START..TAR_CHECKSUM_OFFSET_END] = checksum + checksum = checksum.to_s(8).rjust(6, '0') + header[TAR_CHECKSUM_OFFSET_START..TAR_CHECKSUM_OFFSET_END-2] = checksum + header[TAR_CHECKSUM_OFFSET_END-1] = "\0" return header end From cbef21ff7f683bb84af8b623b35fd8d7b7318c22 Mon Sep 17 00:00:00 2001 From: George Lester Date: Thu, 24 Dec 2015 19:03:02 -0800 Subject: [PATCH 19/30] Trimmed trailing slash on record names --- lib/fpm/package/apk.rb | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 0e0f4f6594..f1f917684e 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -241,7 +241,7 @@ def hash_datatar(target_path) extension_header = header.dup() # directories have a magic string inserted into their name - full_record_path = extension_header[0..(extension_header.index("\0"))] + full_record_path = extension_header[0..99].delete("\0") full_record_path = add_paxstring(full_record_path) # hash data contents with sha1, if there is any content. @@ -251,12 +251,13 @@ def hash_datatar(target_path) # ensure it doesn't end with a slash if(full_record_path[full_record_path.length-1] == '/') - full_record_path = full_record_path[0..full_record_path.length-1] + full_record_path = full_record_path.chop() end else extension_data = hash_record(data) end + logger.info("'#{full_record_path}'") full_record_path = pad_string_to(full_record_path, 100) extension_header[0..99] = full_record_path @@ -415,11 +416,19 @@ def tar_path(path, target_path) # This takes an unchanged directory name and "paxifies" it. def add_paxstring(ret) - pax_slash = ret.rindex('/', ret.rindex('/')-1) - if(!pax_slash || pax_slash < 0) + pax_slash = ret.rindex('/') + if(pax_slash == nil) pax_slash = 0 + else + pax_slash = ret.rindex('/', pax_slash-1) + if(pax_slash == nil || pax_slash < 0) + pax_slash = 0 + end end - return ret.insert(pax_slash, "/PaxHeaders.14670") + + ret = ret.insert(pax_slash, "/PaxHeaders.14670/") + ret = ret.sub("//", "/") + return ret end # Appends null zeroes to the end of [ret] until it is divisible by [length]. From af19256d30409328c09404d18994b606ff019b3d Mon Sep 17 00:00:00 2001 From: George Lester Date: Thu, 24 Dec 2015 19:37:16 -0800 Subject: [PATCH 20/30] Cleaned up header cutting logic, ensured that permission bits are removed from control tar --- lib/fpm/package/apk.rb | 89 ++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index f1f917684e..a9bcf5206e 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -156,55 +156,46 @@ def write_control_scripts(base_path) # Taken together, they comprise 1k of null data. def cut_tar_record(target_path) + temporary_target_path = target_path + "~" + record_length = 0 - contiguous_records = 0 - desired_tar_length = 0 + empty_records = 0 - # Scan to find the location of the two contiguous null records - open(target_path, "rb") do |file| + open(temporary_target_path, "wb") do |target_file| - until(contiguous_records == 2) + # Scan to find the location of the two contiguous null records + open(target_path, "rb") do |file| - # skip to header length - file.read(124) + until(empty_records == 2) - ascii_length = file.read(12) - if(file.eof?()) - raise StandardError.new("Invalid tar stream, eof before end-of-tar record") - end + # skip to header length + header = file.read(512) - record_length = ascii_length.to_i(8) + # clear off ownership info + header = replace_ownership_headers(header) - if(record_length == 0) - contiguous_records += 1 - else - # If there was a previous null tar, add its header length too. - if(contiguous_records != 0) - desired_tar_length += TAR_CHUNK_SIZE + typeflag = header[TAR_TYPEFLAG_OFFSET] + ascii_length = header[TAR_LENGTH_OFFSET_START..TAR_LENGTH_OFFSET_END] + + if(file.eof?()) + raise StandardError.new("Invalid tar stream, eof before end-of-tar record") + end + + if(typeflag == "\0") + empty_records += 1 + next end - # tarballs work in 512-byte blocks, round up to the nearest block. + record_length = ascii_length.to_i(8) record_length = determine_record_length(record_length) - # reset, add length of content and header. - contiguous_records = 0 - desired_tar_length += record_length + TAR_CHUNK_SIZE + target_file.write(header) + target_file.write(file.read(record_length)) end - - # finish off the read of the header length - file.read(376) - - # skip content of record - file.read(record_length) end end - # Truncate file - if(desired_tar_length <= 0) - raise StandardError.new("Unable to trim apk control tar") - end - - File.truncate(target_path, desired_tar_length) + FileUtils::mv(temporary_target_path, target_path) end # Rewrites the tar file located at the given [target_tar_path] @@ -236,6 +227,9 @@ def hash_datatar(target_path) data += file.read(TAR_CHUNK_SIZE) end + # Clear ownership fields + header = replace_ownership_headers(header) + # If it's not a null record, do extension hash. if(typeflag != "\0") extension_header = header.dup() @@ -257,7 +251,6 @@ def hash_datatar(target_path) extension_data = hash_record(data) end - logger.info("'#{full_record_path}'") full_record_path = pad_string_to(full_record_path, 100) extension_header[0..99] = full_record_path @@ -337,9 +330,7 @@ def determine_record_length(record_length) def checksum_header(header) # blank out header checksum - for i in TAR_CHECKSUM_OFFSET_START..TAR_CHECKSUM_OFFSET_END - header[i] = ' ' - end + replace_string_range(header, TAR_CHECKSUM_OFFSET_START, TAR_CHECKSUM_OFFSET_END, ' ') # calculate new checksum checksum = 0 @@ -441,6 +432,28 @@ def pad_string_to(ret, length) return ret end + # Replaces every character between [start] and [finish] in the given [str] + # with [character]. + def replace_string_range(str, start, finish, character) + + for i in (start..finish) + str[i] = character + end + + return str + end + + # Nulls out the ownership bits of the given tar [header]. + def replace_ownership_headers(header) + + header = replace_string_range(header, 108, 115, "\0") #uid + header = replace_string_range(header, 116, 123, "\0") #gid + header = replace_string_range(header, 265, 296, "\0") #uname + header = replace_string_range(header, 297, 328, "\0") #gname + + return header + end + def to_s(format=nil) return super("NAME_FULLVERSION_ARCH.TYPE") if format.nil? return super(format) From 231e4fbee6edbec32a8f8a940f637d4658cc1462 Mon Sep 17 00:00:00 2001 From: George Lester Date: Fri, 25 Dec 2015 00:42:06 -0800 Subject: [PATCH 21/30] Rewrote magic number and usernames in proper places --- lib/fpm/package/apk.rb | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index a9bcf5206e..595b3c4fee 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -172,7 +172,7 @@ def cut_tar_record(target_path) header = file.read(512) # clear off ownership info - header = replace_ownership_headers(header) + header = replace_ownership_headers(header, true) typeflag = header[TAR_TYPEFLAG_OFFSET] ascii_length = header[TAR_LENGTH_OFFSET_START..TAR_LENGTH_OFFSET_END] @@ -228,12 +228,14 @@ def hash_datatar(target_path) end # Clear ownership fields - header = replace_ownership_headers(header) + header = replace_ownership_headers(header, false) # If it's not a null record, do extension hash. if(typeflag != "\0") extension_header = header.dup() + extension_header = replace_ownership_headers(extension_header, true) + # directories have a magic string inserted into their name full_record_path = extension_header[0..99].delete("\0") full_record_path = add_paxstring(full_record_path) @@ -444,12 +446,21 @@ def replace_string_range(str, start, finish, character) end # Nulls out the ownership bits of the given tar [header]. - def replace_ownership_headers(header) + def replace_ownership_headers(header, nullify_names) + + header[257..264] = "ustar\0" + "00" + header = replace_string_range(header, 108, 115, "0") #uid + header = replace_string_range(header, 116, 123, "0") #gid + header[123] = "\0" + header[115] = "\0" - header = replace_string_range(header, 108, 115, "\0") #uid - header = replace_string_range(header, 116, 123, "\0") #gid - header = replace_string_range(header, 265, 296, "\0") #uname - header = replace_string_range(header, 297, 328, "\0") #gname + if(nullify_names) + header = replace_string_range(header, 265, 296, "\0") #uname + header = replace_string_range(header, 297, 328, "\0") #gname + else + header[265..296] = pad_string_to("root", 32) + header[297..328] = pad_string_to("root", 32) + end return header end From 94215c5da06c0e93426c787fa57fda4b548b83db Mon Sep 17 00:00:00 2001 From: George Lester Date: Mon, 28 Dec 2015 14:36:37 -0800 Subject: [PATCH 22/30] Added a few more header fields, 'apk add' still wont accept. --- lib/fpm/package/apk.rb | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 595b3c4fee..65719bc5a9 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -121,13 +121,16 @@ def write_pkginfo(base_path) path = "#{base_path}/.PKGINFO" - pkginfo_io = StringIO::new - package_version = to_s("FULLVERSION") + pkginfo = "" - pkginfo_io << "pkgname = #{@name}\n" - pkginfo_io << "pkgver = #{package_version}\n" + pkginfo << "pkgname = #{@name}\n" + pkginfo << "pkgver = #{to_s("FULLVERSION")}\n" + # pkginfo << "arch = x86_64\n" + # pkginfo << "size = 512\n" + # pkginfo << "pkgdesc = A great package\n" + # pkginfo << "url = http://example.com/fpm\n" - File.write(path, pkginfo_io.string) + File.write(path, pkginfo) end # Writes each control script from template into the build path, @@ -168,7 +171,6 @@ def cut_tar_record(target_path) until(empty_records == 2) - # skip to header length header = file.read(512) # clear off ownership info @@ -395,6 +397,16 @@ def tar_path(path, target_path) "-c" ] + # temporarily move pkginfo to the front. + for i in (0..entries.length) + if(entries[i] == ".PKGINFO") + entries[i] = entries[0] + entries[0] = ".PKGINFO" + break + end + end + + # add entries to arguments. entries.each do |entry| unless(entry == '..' || entry == '.') args = args << entry @@ -448,18 +460,32 @@ def replace_string_range(str, start, finish, character) # Nulls out the ownership bits of the given tar [header]. def replace_ownership_headers(header, nullify_names) + # magic header[257..264] = "ustar\0" + "00" + + # ids header = replace_string_range(header, 108, 115, "0") #uid header = replace_string_range(header, 116, 123, "0") #gid header[123] = "\0" header[115] = "\0" + # names if(nullify_names) header = replace_string_range(header, 265, 296, "\0") #uname header = replace_string_range(header, 297, 328, "\0") #gname + + # major/minor + # header[329..336] = "0".rjust(8, '0') + # header[337..344] = "0".rjust(8, '0') + # header[344] = "\0" + # header[336] = "\0" else header[265..296] = pad_string_to("root", 32) header[297..328] = pad_string_to("root", 32) + + # linkname + #linkname = pad_string_to("\0\0\0APK2", 100) + #header[157..256] = linkname end return header From 4418f1af1be6395c5a848adf32e936ae2c2ecc07 Mon Sep 17 00:00:00 2001 From: George Lester Date: Mon, 28 Dec 2015 14:36:57 -0800 Subject: [PATCH 23/30] Added debug echoes to templates to determine when hooks are used --- templates/apk/post-install | 1 + templates/apk/pre-install | 1 + 2 files changed, 2 insertions(+) diff --git a/templates/apk/post-install b/templates/apk/post-install index 039e4d0069..4b89f7f198 100644 --- a/templates/apk/post-install +++ b/templates/apk/post-install @@ -1,2 +1,3 @@ #!/bin/sh +echo "post-install" exit 0 diff --git a/templates/apk/pre-install b/templates/apk/pre-install index 039e4d0069..32d095e9ab 100644 --- a/templates/apk/pre-install +++ b/templates/apk/pre-install @@ -1,2 +1,3 @@ #!/bin/sh +echo "pre-install" exit 0 From 15ac8a08370320e7f63ba74d7971f184a256b719 Mon Sep 17 00:00:00 2001 From: George Lester Date: Mon, 28 Dec 2015 16:46:02 -0800 Subject: [PATCH 24/30] First actually-working version --- lib/fpm/package/apk.rb | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 65719bc5a9..9b46d31947 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -60,6 +60,13 @@ def prefix return (attributes[:prefix] or "/") end # def prefix + def architecture + if @architecture.nil? or @architecture == "native" + @architecture = "noarch" + end + return @architecture + end # def architecture + def input(input_path) extract_info(input_path) extract_files(input_path) @@ -110,27 +117,22 @@ def output(output_path) # concatenate the two into the final apk concat_zip_tars(controltar_path, datatar_path, output_path) - ensure - logger.warn("apk output to is not implemented") - `rm -rf /tmp/apkfpm` - `cp -r #{build_path("")} /tmp/apkfpm` end end def write_pkginfo(base_path) - path = "#{base_path}/.PKGINFO" - pkginfo = "" + pkginfo << "# Generated by fpm\n" pkginfo << "pkgname = #{@name}\n" pkginfo << "pkgver = #{to_s("FULLVERSION")}\n" - # pkginfo << "arch = x86_64\n" - # pkginfo << "size = 512\n" - # pkginfo << "pkgdesc = A great package\n" - # pkginfo << "url = http://example.com/fpm\n" + pkginfo << "arch = x86_64\n" + pkginfo << "size = 102400\n" + pkginfo << "pkgdesc = A great package\n" + pkginfo << "url = http://example.com/fpm\n" - File.write(path, pkginfo) + File.write("#{base_path}/.PKGINFO", pkginfo) end # Writes each control script from template into the build path, From 9b3c0f3bf0bbb86c8edbc387188f79dac96689de Mon Sep 17 00:00:00 2001 From: George Lester Date: Mon, 28 Dec 2015 16:47:28 -0800 Subject: [PATCH 25/30] Uncommented major/minor tar version clearing, and removed legacy checksum linkname stuff --- lib/fpm/package/apk.rb | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 9b46d31947..3ca5473476 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -399,7 +399,7 @@ def tar_path(path, target_path) "-c" ] - # temporarily move pkginfo to the front. + # Move pkginfo to the front, if it exists. for i in (0..entries.length) if(entries[i] == ".PKGINFO") entries[i] = entries[0] @@ -477,17 +477,13 @@ def replace_ownership_headers(header, nullify_names) header = replace_string_range(header, 297, 328, "\0") #gname # major/minor - # header[329..336] = "0".rjust(8, '0') - # header[337..344] = "0".rjust(8, '0') - # header[344] = "\0" - # header[336] = "\0" + header[329..336] = "0".rjust(8, '0') + header[337..344] = "0".rjust(8, '0') + header[344] = "\0" + header[336] = "\0" else header[265..296] = pad_string_to("root", 32) header[297..328] = pad_string_to("root", 32) - - # linkname - #linkname = pad_string_to("\0\0\0APK2", 100) - #header[157..256] = linkname end return header From 9e857deabd8c971f55ba8dc0900f0aaebcbe3e52 Mon Sep 17 00:00:00 2001 From: George Lester Date: Mon, 28 Dec 2015 16:58:03 -0800 Subject: [PATCH 26/30] Removed more magic constants, and added warning about using --allow-untrusted --- lib/fpm/package/apk.rb | 56 +++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 3ca5473476..533d3b4b52 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -15,6 +15,8 @@ class FPM::Package::APK< FPM::Package TAR_CHUNK_SIZE = 512 TAR_TYPEFLAG_OFFSET = 156 + TAR_NAME_OFFSET_START = 0 + TAR_NAME_OFFSET_END = 99 TAR_LENGTH_OFFSET_START = 124 TAR_LENGTH_OFFSET_END = 135 TAR_CHECKSUM_OFFSET_START = 148 @@ -118,6 +120,9 @@ def output(output_path) # concatenate the two into the final apk concat_zip_tars(controltar_path, datatar_path, output_path) end + + logger.warn("apk output does not currently sign packages.") + logger.warn("It's recommended that your package be installed with '--allow-untrusted'") end def write_pkginfo(base_path) @@ -241,7 +246,7 @@ def hash_datatar(target_path) extension_header = replace_ownership_headers(extension_header, true) # directories have a magic string inserted into their name - full_record_path = extension_header[0..99].delete("\0") + full_record_path = extension_header[TAR_NAME_OFFSET_START..TAR_NAME_OFFSET_END].delete("\0") full_record_path = add_paxstring(full_record_path) # hash data contents with sha1, if there is any content. @@ -258,7 +263,7 @@ def hash_datatar(target_path) end full_record_path = pad_string_to(full_record_path, 100) - extension_header[0..99] = full_record_path + extension_header[TAR_NAME_OFFSET_START..TAR_NAME_OFFSET_END] = full_record_path extension_header[TAR_TYPEFLAG_OFFSET] = 'x' extension_header[TAR_LENGTH_OFFSET_START..TAR_LENGTH_OFFSET_END] = extension_data.length.to_s(8).rjust(12, '0') @@ -323,8 +328,10 @@ def concat_zip_tars(apath, bpath, target_path) # Rounds the given [record_length] to the nearest highest evenly-divisble number of 512. def determine_record_length(record_length) - if(record_length % 512 != 0) - record_length = (record_length + 511) & ~511; + sans_size = TAR_CHUNK_SIZE-1 + + if(record_length % TAR_CHUNK_SIZE != 0) + record_length = (record_length + sans_size) & ~sans_size; end return record_length end @@ -462,28 +469,43 @@ def replace_string_range(str, start, finish, character) # Nulls out the ownership bits of the given tar [header]. def replace_ownership_headers(header, nullify_names) + TAR_MAGIC_START = 257 + TAR_MAGIC_END = 264 + TAR_UID_START = 108 + TAR_UID_END = 115 + TAR_GID_START = 116 + TAR_GID_END = 123 + TAR_UNAME_START = 265 + TAR_UNAME_END = 296 + TAR_GNAME_START = 297 + TAR_GNAME_END = 328 + TAR_MAJOR_START = 329 + TAR_MAJOR_END = 336 + TAR_MINOR_START = 337 + TAR_MINOR_END = 344 + # magic - header[257..264] = "ustar\0" + "00" + header[TAR_MAGIC_START..TAR_MAGIC_END] = "ustar\0" + "00" # ids - header = replace_string_range(header, 108, 115, "0") #uid - header = replace_string_range(header, 116, 123, "0") #gid - header[123] = "\0" - header[115] = "\0" + header = replace_string_range(header, TAR_UID_START, TAR_UID_END, "0") + header = replace_string_range(header, TAR_GID_START, TAR_GID_END, "0") + header[TAR_GID_END] = "\0" + header[TAR_UID_END] = "\0" # names if(nullify_names) - header = replace_string_range(header, 265, 296, "\0") #uname - header = replace_string_range(header, 297, 328, "\0") #gname + header = replace_string_range(header, TAR_UNAME_START, TAR_UNAME_END, "\0") + header = replace_string_range(header, TAR_GNAME_START, TAR_GNAME_END, "\0") # major/minor - header[329..336] = "0".rjust(8, '0') - header[337..344] = "0".rjust(8, '0') - header[344] = "\0" - header[336] = "\0" + header[TAR_MAJOR_START..TAR_MAJOR_END] = "0".rjust(8, '0') + header[TAR_MINOR_START..TAR_MINOR_END] = "0".rjust(8, '0') + header[TAR_MAJOR_END] = "\0" + header[TAR_MINOR_END] = "\0" else - header[265..296] = pad_string_to("root", 32) - header[297..328] = pad_string_to("root", 32) + header[TAR_UNAME_START..TAR_UNAME_END] = pad_string_to("root", 32) + header[TAR_GNAME_START..TAR_GNAME_END] = pad_string_to("root", 32) end return header From 333a2d6f5d5161878a0c99463e9f48b0645534ac Mon Sep 17 00:00:00 2001 From: George Lester Date: Mon, 28 Dec 2015 17:07:12 -0800 Subject: [PATCH 27/30] Moved constants to top, and properly used fpm-provided arch/url/description --- lib/fpm/package/apk.rb | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 533d3b4b52..0db8142732 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -21,6 +21,20 @@ class FPM::Package::APK< FPM::Package TAR_LENGTH_OFFSET_END = 135 TAR_CHECKSUM_OFFSET_START = 148 TAR_CHECKSUM_OFFSET_END = 155 + TAR_MAGIC_START = 257 + TAR_MAGIC_END = 264 + TAR_UID_START = 108 + TAR_UID_END = 115 + TAR_GID_START = 116 + TAR_GID_END = 123 + TAR_UNAME_START = 265 + TAR_UNAME_END = 296 + TAR_GNAME_START = 297 + TAR_GNAME_END = 328 + TAR_MAJOR_START = 329 + TAR_MAJOR_END = 336 + TAR_MINOR_START = 337 + TAR_MINOR_END = 344 # Map of what scripts are named. SCRIPT_MAP = { @@ -132,10 +146,10 @@ def write_pkginfo(base_path) pkginfo << "# Generated by fpm\n" pkginfo << "pkgname = #{@name}\n" pkginfo << "pkgver = #{to_s("FULLVERSION")}\n" - pkginfo << "arch = x86_64\n" - pkginfo << "size = 102400\n" - pkginfo << "pkgdesc = A great package\n" - pkginfo << "url = http://example.com/fpm\n" + pkginfo << "arch = #{architecture()}\n" + pkginfo << "pkgdesc = #{description()}\n" + pkginfo << "url = #{url()}\n" + pkginfo << "size = 102400\n" # totally magic, not sure what it's used for. File.write("#{base_path}/.PKGINFO", pkginfo) end @@ -178,7 +192,7 @@ def cut_tar_record(target_path) until(empty_records == 2) - header = file.read(512) + header = file.read(TAR_CHUNK_SIZE) # clear off ownership info header = replace_ownership_headers(header, true) @@ -469,21 +483,6 @@ def replace_string_range(str, start, finish, character) # Nulls out the ownership bits of the given tar [header]. def replace_ownership_headers(header, nullify_names) - TAR_MAGIC_START = 257 - TAR_MAGIC_END = 264 - TAR_UID_START = 108 - TAR_UID_END = 115 - TAR_GID_START = 116 - TAR_GID_END = 123 - TAR_UNAME_START = 265 - TAR_UNAME_END = 296 - TAR_GNAME_START = 297 - TAR_GNAME_END = 328 - TAR_MAJOR_START = 329 - TAR_MAJOR_END = 336 - TAR_MINOR_START = 337 - TAR_MINOR_END = 344 - # magic header[TAR_MAGIC_START..TAR_MAGIC_END] = "ustar\0" + "00" From 8ac30f4daf8a2746d497ecd8eed90ed9bde41d52 Mon Sep 17 00:00:00 2001 From: George Lester Date: Mon, 28 Dec 2015 17:10:24 -0800 Subject: [PATCH 28/30] Removed unused constants, and cleaned up namespace errors with 'Dir' --- lib/fpm/package/apk.rb | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 0db8142732..510c2f03a5 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -36,17 +36,6 @@ class FPM::Package::APK< FPM::Package TAR_MINOR_START = 337 TAR_MINOR_END = 344 - # Map of what scripts are named. - SCRIPT_MAP = { - :before_install => "pre-install", - :after_install => "post-install", - :before_remove => "pre-deinstall", - :after_remove => "post-deinstall", - } unless defined?(SCRIPT_MAP) - - # The list of supported compression types. Default is gz (gzip) - COMPRESSION_TYPES = [ "gz" ] - private # Get the name of this package. See also FPM::Package#name @@ -70,35 +59,24 @@ def name end return @name - end # def name + end def prefix return (attributes[:prefix] or "/") - end # def prefix + end def architecture + + # "native" in apk should be "noarch" if @architecture.nil? or @architecture == "native" @architecture = "noarch" end return @architecture - end # def architecture + end def input(input_path) - extract_info(input_path) - extract_files(input_path) - end # def input - - def extract_info(package) - - logger.error("Extraction is not yet implemented") - end # def extract_info - - def extract_files(package) - - # unpack the data.tar.{gz,bz2,xz} from the deb package into staging_path - safesystem("ar p #{package} data.tar.gz " \ - "| tar gz -xf - -C #{staging_path}") - end # def extract_files + logger.error("apk extraction is not yet implemented") + end def output(output_path) @@ -409,8 +387,8 @@ def tar_path(path, target_path) # Change directory to the source path, and glob files # This is done so that we end up with a "flat" archive, that doesn't # have any path artifacts from the packager's absolute path. - File::Dir::chdir(path) do - entries = File::Dir::glob("**", File::FNM_DOTMATCH) + ::Dir::chdir(path) do + entries = ::Dir::glob("**", File::FNM_DOTMATCH) args = [ From 277926b8f93c16f6b3cef521d61b09bdf9f9474e Mon Sep 17 00:00:00 2001 From: George Lester Date: Mon, 28 Dec 2015 17:21:50 -0800 Subject: [PATCH 29/30] Implemented dependency-specifying support --- lib/fpm/package/apk.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 510c2f03a5..5f4d5216ad 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -129,6 +129,11 @@ def write_pkginfo(base_path) pkginfo << "url = #{url()}\n" pkginfo << "size = 102400\n" # totally magic, not sure what it's used for. + # write depends lines + for dependency in dependencies() + pkginfo << "depend = #{dependency}\n" + end + File.write("#{base_path}/.PKGINFO", pkginfo) end From a2540649fcb9e433dfb7f1d623152e407d92f4a6 Mon Sep 17 00:00:00 2001 From: George Lester Date: Mon, 28 Dec 2015 17:56:54 -0800 Subject: [PATCH 30/30] Implemented control script commands, removed dummy template scripts --- lib/fpm/package/apk.rb | 36 ++++++++++++++++++++++-------------- templates/apk/post-deinstall | 2 -- templates/apk/post-install | 3 --- templates/apk/post-upgrade | 2 -- templates/apk/pre-deinstall | 2 -- templates/apk/pre-install | 3 --- templates/apk/pre-upgrade | 2 -- 7 files changed, 22 insertions(+), 28 deletions(-) delete mode 100644 templates/apk/post-deinstall delete mode 100644 templates/apk/post-install delete mode 100644 templates/apk/post-upgrade delete mode 100644 templates/apk/pre-deinstall delete mode 100644 templates/apk/pre-install delete mode 100644 templates/apk/pre-upgrade diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb index 5f4d5216ad..aa0421e8e1 100644 --- a/lib/fpm/package/apk.rb +++ b/lib/fpm/package/apk.rb @@ -141,21 +141,29 @@ def write_pkginfo(base_path) # in the folder given by [base_path] def write_control_scripts(base_path) - scripts = - [ - "pre-install", - "post-install", - "pre-deinstall", - "post-deinstall", - "pre-upgrade", - "post-upgrade" - ] - - scripts.each do |path| - - script_path = "#{base_path}/.#{path}" - File.write(script_path, template("apk/#{path}").result(binding)) + scripts = {} + + scripts = register_script('post-install', :after_install, scripts) + scripts = register_script('post-install', :before_install, scripts) + scripts = register_script('post-install', :before_upgrade, scripts) + scripts = register_script('post-install', :after_upgrade, scripts) + scripts = register_script('pre-deinstall', :before_remove, scripts) + scripts = register_script('post-deinstall', :after_remove, scripts) + + scripts.each do |key, content| + + File.write("#{base_path}/.#{key}", content) + end + end + + # Convenience method for 'write_control_scripts' to register control scripts + # if they exist. + def register_script(key, value, hash) + + if(script?(value)) + hash[key] = scripts[value] end + return hash end # Removes the end-of-tar records from the given [target_path]. diff --git a/templates/apk/post-deinstall b/templates/apk/post-deinstall deleted file mode 100644 index 039e4d0069..0000000000 --- a/templates/apk/post-deinstall +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exit 0 diff --git a/templates/apk/post-install b/templates/apk/post-install deleted file mode 100644 index 4b89f7f198..0000000000 --- a/templates/apk/post-install +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -echo "post-install" -exit 0 diff --git a/templates/apk/post-upgrade b/templates/apk/post-upgrade deleted file mode 100644 index 039e4d0069..0000000000 --- a/templates/apk/post-upgrade +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exit 0 diff --git a/templates/apk/pre-deinstall b/templates/apk/pre-deinstall deleted file mode 100644 index 039e4d0069..0000000000 --- a/templates/apk/pre-deinstall +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exit 0 diff --git a/templates/apk/pre-install b/templates/apk/pre-install deleted file mode 100644 index 32d095e9ab..0000000000 --- a/templates/apk/pre-install +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -echo "pre-install" -exit 0 diff --git a/templates/apk/pre-upgrade b/templates/apk/pre-upgrade deleted file mode 100644 index 039e4d0069..0000000000 --- a/templates/apk/pre-upgrade +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exit 0