-
-
Notifications
You must be signed in to change notification settings - Fork 605
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enhanced scripts/build and related Python scripts to support export f…
…iles that make up an image versus uploading them to an image This patch enhances OSv build scripts to allow exporting files in addition of uploading them to an image. It addresses 3 usage scenarios described in #900 and demostrated by examples below: - ./scripts/build image=empty export=all - export files that are specified is usr.manifest.skel in order to be able to produce capstan osv.bootstrap package - ./scripts/build image=java-isolated export=all usrskel=none - export files that are part of OSv modules (./modules directory) along with all other dependent modules minus what is part of usr.manifest.skel - ./scripts/build image=openjdk8-zulu-compact1 export=selected usrskel=none - export files that are part of an app (most applicable to apps/openjdk8-****) without any dependent modules The export logic is enabled by passing new parameter export [=none|all|selected] to scripts/build. Exported files are placed under build/export directory or directory indocated by export_dir parameter. Please note that the changes are backwards compatible. However changes to scripts/upload_manifest.py will break patches that are part of https://github.com/mikelangelo-project/capstan-packages/tree/master/docker_files/common. Hopefully this patch will make some of these patches obsolete. Fixed #900 Signed-off-by: Waldemar Kozaczuk <[email protected]> Message-Id: <[email protected]>
- Loading branch information
Showing
6 changed files
with
198 additions
and
82 deletions.
There are no files selected for viewing
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| #!/usr/bin/python | ||
|
|
||
| import optparse, os, shutil | ||
| from manifest_common import add_var, expand, unsymlink, read_manifest, defines, strip_file | ||
|
|
||
| # This will export the package based on the provided manifest file. It uses the same mechanism to | ||
| # get the files that need copying as the actual upload process. The only current limitation is | ||
| # support for links in OSv, e.g., /etc/mnttab: ->/proc/mounts. | ||
| def export_package(manifest, dest): | ||
| abs_dest = os.path.abspath(dest) | ||
| print "[INFO] exporting into directory %s" % abs_dest | ||
|
|
||
| # Remove and create the base directory where we are going to put all package files. | ||
| if os.path.exists(abs_dest): | ||
| shutil.rmtree(abs_dest) | ||
| os.makedirs(abs_dest) | ||
|
|
||
| files = list(expand(manifest)) | ||
| files = [(x, unsymlink(y % defines)) for (x, y) in files] | ||
|
|
||
| for name, hostname in files: | ||
| name = name[1:] if name.startswith("/") else name | ||
| name = os.path.join(abs_dest, name) | ||
|
|
||
| if hostname.startswith("->"): | ||
| link_source = hostname[2:] | ||
| target_dir = os.path.dirname(name) | ||
|
|
||
| if link_source.startswith("/"): | ||
| link_source = os.path.join(abs_dest, link_source[1:]) | ||
| else: | ||
| link_source = os.path.abspath(os.path.join(target_dir, link_source)) | ||
|
|
||
| link_source = os.path.relpath(link_source, target_dir) | ||
|
|
||
| if not os.path.exists(target_dir): | ||
| os.makedirs(target_dir) | ||
|
|
||
| os.symlink(link_source, name) | ||
| print "[INFO] added link %s -> %s" % (name, link_source) | ||
|
|
||
| else: | ||
| # If it is a file, copy it to the target directory. | ||
| if os.path.isfile(hostname): | ||
| # Make sure the target dir exists | ||
| dirname = os.path.dirname(name) | ||
| if not os.path.exists(dirname): | ||
| os.makedirs(dirname) | ||
|
|
||
| if hostname.endswith("-stripped.so"): | ||
| continue | ||
| hostname = strip_file(hostname) | ||
|
|
||
| shutil.copy(hostname, name) | ||
| print "[INFO] exported %s" % name | ||
| elif os.path.isdir(hostname): | ||
| # If hostname is a dir, it is only a request to create the folder on guest. Nothing to copy. | ||
| if not os.path.exists(name): | ||
| os.makedirs(name) | ||
| print "[INFO] created dir %s" % name | ||
|
|
||
| else: | ||
| # Inform the user that the rule cannot be applied. For example, this happens for links in OSv. | ||
| print "[ERR] unable to export %s" % hostname | ||
|
|
||
|
|
||
| def main(): | ||
| make_option = optparse.make_option | ||
|
|
||
| opt = optparse.OptionParser(option_list=[ | ||
| make_option('-m', | ||
| dest='manifest', | ||
| help='read manifest from FILE', | ||
| metavar='FILE'), | ||
| make_option('-D', | ||
| type='string', | ||
| help='define VAR=DATA', | ||
| metavar='VAR=DATA', | ||
| action='callback', | ||
| callback=add_var), | ||
| make_option('-e', | ||
| dest='export', | ||
| help='exports the contents of the usr.manifest into a given folder', | ||
| metavar='FILE'), | ||
| ]) | ||
|
|
||
| (options, args) = opt.parse_args() | ||
|
|
||
| manifest = read_manifest(options.manifest) | ||
| export_package(manifest, options.export) | ||
|
|
||
| if __name__ == "__main__": | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| #!/usr/bin/python | ||
|
|
||
| import os, io, re, subprocess | ||
|
|
||
| defines = {} | ||
|
|
||
| def add_var(option, opt, value, parser): | ||
| var, val = value.split('=') | ||
| defines[var] = val | ||
|
|
||
| def expand(items): | ||
| for name, hostname in items: | ||
| if name.endswith('/**') and hostname.endswith('/**'): | ||
| name = name[:-2] | ||
| hostname = hostname[:-2] | ||
| for dirpath, dirnames, filenames in os.walk(hostname): | ||
| for filename in filenames: | ||
| relpath = dirpath[len(hostname):] | ||
| if relpath != "": | ||
| relpath += "/" | ||
| yield (name + relpath + filename, | ||
| hostname + relpath + filename) | ||
| elif '/&/' in name and hostname.endswith('/&'): | ||
| prefix, suffix = name.split('/&/', 1) | ||
| yield (prefix + '/' + suffix, hostname[:-1] + suffix) | ||
| else: | ||
| yield (name, hostname) | ||
|
|
||
| def unsymlink(f): | ||
| if f.startswith('!'): | ||
| return f[1:] | ||
| if f.startswith('->'): | ||
| return f | ||
| try: | ||
| link = os.readlink(f) | ||
| if link.startswith('/'): | ||
| # try to find a match | ||
| base = os.path.dirname(f) | ||
| while not os.path.exists(base + link): | ||
| if base == '/': | ||
| return f | ||
| base = os.path.dirname(base) | ||
| else: | ||
| base = os.path.dirname(f) + '/' | ||
| return unsymlink(base + link) | ||
| except Exception: | ||
| return f | ||
|
|
||
| # Reads the manifest and returns it as a list of pairs (guestpath, hostpath). | ||
| def read_manifest(fn): | ||
| ret = [] | ||
| comment = re.compile("^[ \t]*(|#.*|\[manifest])$") | ||
| with open(fn, 'r') as f: | ||
| for line in f: | ||
| line = line.rstrip(); | ||
| if comment.match(line): continue | ||
| components = line.split(": ", 2) | ||
| guestpath = components[0].strip(); | ||
| hostpath = components[1].strip() | ||
| ret.append((guestpath, hostpath)) | ||
| return ret | ||
|
|
||
| def strip_file(filename): | ||
| def to_strip(filename): | ||
| ff = os.path.abspath(filename) | ||
| osvdir = os.path.abspath('../..') | ||
| return ff.startswith(os.getcwd()) or \ | ||
| ff.startswith(osvdir + "/modules") or \ | ||
| ff.startswith(osvdir + "/apps") | ||
|
|
||
| stripped_filename = filename | ||
| if filename.endswith(".so") and to_strip(filename): | ||
| stripped_filename = filename[:-3] + "-stripped.so" | ||
| if not os.path.exists(stripped_filename) \ | ||
| or (os.path.getmtime(stripped_filename) < \ | ||
| os.path.getmtime(filename)): | ||
| subprocess.call(["strip", "-o", stripped_filename, filename]) | ||
| return stripped_filename |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters