Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[master] add docker-buildx-plugin package (deb, rpm, static) #654

Merged
merged 3 commits into from
Mar 27, 2022
Merged

[master] add docker-buildx-plugin package (deb, rpm, static) #654

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
37aa0a7
add docker-buildx-plugin package (deb, rpm)
crazy-max Mar 27, 2022
97b6ad8
add docker-buildx-plugin static package
crazy-max Mar 26, 2022
4f48724
remove obsolete plugin installers
crazy-max Mar 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ clean-src:
$(RM) -r src

.PHONY: src
src: src/github.com/docker/cli src/github.com/docker/docker src/github.com/docker/compose src/github.com/docker/scan-cli-plugin ## clone source
src: src/github.com/docker/cli src/github.com/docker/docker src/github.com/docker/buildx src/github.com/docker/compose src/github.com/docker/scan-cli-plugin ## clone source

ifdef CLI_DIR
src/github.com/docker/cli:
Expand All @@ -37,6 +37,10 @@ src/github.com/docker/docker:
git -C $@ remote add origin "$(DOCKER_ENGINE_REPO)"
endif

src/github.com/docker/buildx:
git init $@
git -C $@ remote add origin "$(DOCKER_BUILDX_REPO)"

src/github.com/docker/compose:
git init $@
git -C $@ remote add origin "$(DOCKER_COMPOSE_REPO)"
Expand All @@ -54,6 +58,10 @@ checkout-cli: src/github.com/docker/cli
checkout-docker: src/github.com/docker/docker
./scripts/checkout.sh src/github.com/docker/docker "$(DOCKER_ENGINE_REF)"

.PHONY: checkout-buildx
checkout-buildx: src/github.com/docker/buildx
./scripts/checkout.sh src/github.com/docker/buildx "$(DOCKER_BUILDX_REF)"

.PHONY: checkout-compose
checkout-compose: src/github.com/docker/compose
./scripts/checkout.sh src/github.com/docker/compose "$(DOCKER_COMPOSE_REF)"
Expand All @@ -63,7 +71,7 @@ checkout-scan-cli-plugin: src/github.com/docker/scan-cli-plugin
./scripts/checkout.sh src/github.com/docker/scan-cli-plugin "$(DOCKER_SCAN_REF)"

.PHONY: checkout
checkout: checkout-cli checkout-docker checkout-compose checkout-scan-cli-plugin ## checkout source at the given reference(s)
checkout: checkout-cli checkout-docker checkout-buildx checkout-compose checkout-scan-cli-plugin ## checkout source at the given reference(s)

.PHONY: clean
clean: clean-src ## remove build artifacts
Expand Down
2 changes: 2 additions & 0 deletions common.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ DOCKER_CLI_REPO ?= https://github.com/docker/cli.git
DOCKER_ENGINE_REPO ?= https://github.com/docker/docker.git
DOCKER_SCAN_REPO ?= https://github.com/docker/scan-cli-plugin.git
DOCKER_COMPOSE_REPO ?= https://github.com/docker/compose.git
DOCKER_BUILDX_REPO ?= https://github.com/docker/buildx.git

# REF can be used to specify the same branch or tag to use for *both* the CLI
# and Engine source code. This can be useful if both the CLI and Engine have a
Expand All @@ -40,6 +41,7 @@ DOCKER_CLI_REF ?= $(REF)
DOCKER_ENGINE_REF ?= $(REF)
DOCKER_SCAN_REF ?= v0.17.0
DOCKER_COMPOSE_REF ?= v2.3.4
DOCKER_BUILDX_REF ?= v0.8.1

export BUILDTIME
export DEFAULT_PRODUCT_LICENSE
Expand Down
18 changes: 10 additions & 8 deletions deb/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ RUN?=docker run --rm \
-e VERSION=$(word 2, $(GEN_DEB_VER)) \
-e CLI_GITCOMMIT=$(CLI_GITCOMMIT) \
-e ENGINE_GITCOMMIT=$(ENGINE_GITCOMMIT) \
-e BUILDX_VERSION=$(DOCKER_BUILDX_REF) \
-e COMPOSE_VERSION=$(DOCKER_COMPOSE_REF) \
-e SCAN_VERSION=$(DOCKER_SCAN_REF) \
-e SCAN_GITCOMMIT=$(SCAN_GITCOMMIT) \
Expand Down Expand Up @@ -78,7 +79,7 @@ $(DISTROS): sources
$(CHOWN) -R $(shell id -u):$(shell id -g) "debbuild/$@"

.PHONY: sources
sources: sources/cli.tgz sources/engine.tgz sources/compose.tgz sources/scan-cli-plugin.tgz sources/plugin-installers.tgz
sources: sources/cli.tgz sources/engine.tgz sources/buildx.tgz sources/compose.tgz sources/scan-cli-plugin.tgz

sources/engine.tgz:
mkdir -p $(@D)
Expand All @@ -96,6 +97,14 @@ sources/cli.tgz:
alpine \
tar -C / -c -z -f /v/cli.tgz --exclude .git cli

sources/buildx.tgz:
mkdir -p $(@D)
docker run --rm -w /v \
-v $(realpath $(CURDIR)/../src/github.com/docker/buildx):/buildx \
-v $(CURDIR)/$(@D):/v \
alpine \
tar -C / -c -z -f /v/buildx.tgz --exclude .git buildx

sources/compose.tgz:
mkdir -p $(@D)
docker run --rm -w /v \
Expand All @@ -112,13 +121,6 @@ sources/scan-cli-plugin.tgz:
alpine \
tar -C / -c -z -f /v/scan-cli-plugin.tgz --exclude .git scan-cli-plugin

sources/plugin-installers.tgz: $(wildcard ../plugins/*)
docker run --rm -w /v \
-v $(PLUGINS_DIR):/plugins \
-v $(CURDIR)/$(@D):/v \
alpine \
tar -C / -c -z -f /v/plugin-installers.tgz --exclude .git plugins

# See ARCHES in common.mk. Could not figure out how to match both distro and arch.
BUNDLES:=$(addsuffix .tar.gz,$(addprefix debbuild/bundles-ce-%-,$(ARCHES)))

Expand Down
3 changes: 3 additions & 0 deletions deb/build-deb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ mkdir -p /root/build-deb/engine
tar -C /root/build-deb -xzf /sources/engine.tgz
mkdir -p /root/build-deb/cli
tar -C /root/build-deb -xzf /sources/cli.tgz
mkdir -p /root/build-deb/buildx
tar -C /root/build-deb -xzf /sources/buildx.tgz
mkdir -p /root/build-deb/compose
tar -C /root/build-deb -xzf /sources/compose.tgz
mkdir -p /root/build-deb/scan-cli-plugin
Expand All @@ -16,6 +18,7 @@ tar -C /root/build-deb -xzf /sources/scan-cli-plugin.tgz
mkdir -p /go/src/github.com/docker
ln -snf /root/build-deb/engine /go/src/github.com/docker/docker
ln -snf /root/build-deb/cli /go/src/github.com/docker/cli
ln -snf /root/build-deb/buildx /go/src/github.com/docker/buildx
ln -snf /root/build-deb/compose /go/src/github.com/docker/compose
ln -snf /root/build-deb/scan-cli-plugin /go/src/github.com/docker/scan-cli-plugin

Expand Down
11 changes: 10 additions & 1 deletion deb/common/control
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ Package: docker-ce-cli
Architecture: linux-any
Depends: ${shlibs:Depends}
# TODO change once we support scan-plugin on other architectures
Recommends: docker-compose-plugin,
Recommends: docker-buildx-plugin,
docker-compose-plugin,
docker-scan-plugin [amd64]
Conflicts: docker (<< 1.5~),
docker-engine,
Expand Down Expand Up @@ -103,6 +104,14 @@ Description: Rootless support for Docker.
Either VPNKit or slirp4netns (>= 0.4.0) needs to be installed separately.
Homepage: https://docs.docker.com/engine/security/rootless/

Package: docker-buildx-plugin
Priority: optional
Replaces: docker-ce-cli
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I initially was confused by the Replaces: here, as I thought it would mark this package to replace the whole docker-ce-cli package, but @crazy-max explained this also can be used to replace some files (given, it's a bit "fuzzy", as it doesn't require speciyfing which files), from the The Debian Administrator's Handbook:

5.2.1.5. Replacing Files: The Replaces Field

The Replaces field indicates that the package contains files that are also present in another package, but that the package is legitimately entitled to replace them. Without this specification, dpkg fails, stating that it can not overwrite the files of another package (technically, it is possible to force it to do so with the --force-overwrite option, but that is not considered standard operation). This allows identification of potential problems and requires the maintainer to study the matter prior to choosing whether to add such a field.

The use of this field is justified when package names change or when a package is included in another. This also happens when the maintainer decides to distribute files differently among various binary packages produced from the same source package: a replaced file no longer belongs to the old package, but only to the new one.

If all of the files in an installed package have been replaced, the package is considered to be removed. Finally, this field also encourages dpkg to remove the replaced package where there is a conflict.

The last paragraph outlines our situation

What we probably could consider doing is (if supported) to include the version here, e.g.

Suggested change
Replaces: docker-ce-cli
Replaces: docker-ce-cli (<< 22.04.0)

But we need to check if it likes CalVer; instead, probably this would work;

Suggested change
Replaces: docker-ce-cli
Replaces: docker-ce-cli (<= 21.10~)

Copy link
Member Author

@crazy-max crazy-max Mar 27, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unfortunately I tried and it doesn't work with CalVer. I think it's fine to keep docker-ce-cli as long as the docker-buildx-plugin package only replaces one file from it.

Copy link
Member Author

@crazy-max crazy-max Mar 27, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

About relationships see also https://www.debian.org/doc/debian-policy/ch-relationships.html#overwriting-files-and-replacing-packages-replaces

It is usually an error for a package to contain files which are on the system in another package. However, if the overwriting package declares that it Replaces the one containing the file being overwritten, then dpkg will replace the file from the old package with that from the new. The file will no longer be listed as “owned” by the old package and will be taken over by the new package. Normally, Breaks should be used in conjunction with Replaces.

Architecture: linux-any
Enhances: docker-ce-cli
Description: Docker Buildx cli plugin.
Homepage: https://github.com/docker/buildx

Package: docker-compose-plugin
Priority: optional
Architecture: linux-any
Expand Down
31 changes: 16 additions & 15 deletions deb/common/rules
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ override_dh_auto_build:
# Build the CLI
cd /go/src/github.com/docker/cli && VERSION=$(VERSION) GITCOMMIT=$(CLI_GITCOMMIT) LDFLAGS='' GO_LINKMODE=dynamic ./scripts/build/binary && DISABLE_WARN_OUTSIDE_CONTAINER=1 LDFLAGS='' make manpages

# Build buildx plugin
cd /go/src/github.com/docker/buildx \
&& mkdir -p /usr/libexec/docker/cli-plugins/ \
&& CGO_ENABLED=0 GO111MODULE=on go build -mod=vendor -o /usr/libexec/docker/cli-plugins/docker-buildx -ldflags "-X github.com/docker/buildx/version.Version=$(BUILDX_VERSION) -X github.com/docker/buildx/version.Revision=$(git rev-parse HEAD) -X github.com/docker/buildx/version.Package=github.com/docker/buildx" ./cmd/buildx

# Build the compose plugin
# FIXME: using GOPROXY, to work around:
# go: github.com/Azure/[email protected]+incompatible: reading github.com/Azure/azure-sdk-for-go/go.mod at revision v48.2.0: unknown revision v48.2.0
Expand All @@ -34,21 +39,16 @@ override_dh_auto_build:
&& mv bin/docker-scan /usr/libexec/docker/cli-plugins/; \
fi

# Build the CLI plugins
# Make sure to set LDFLAGS="" since, dpkg-buildflags sets it to some weird values
set -e;cd /sources && \
tar xzf plugin-installers.tgz; \
for installer in plugins/*.installer; do \
LDFLAGS='' bash $${installer} build; \
done

override_dh_auto_test:
ver="$$(engine/bundles/dynbinary-daemon/dockerd --version)"; \
test "$$ver" = "Docker version $(VERSION), build $(ENGINE_GITCOMMIT)" && echo "PASS: daemon version OK" || (echo "FAIL: daemon version ($$ver) did not match" && exit 1)

ver="$$(cli/build/docker --version)"; \
test "$$ver" = "Docker version $(VERSION), build $(CLI_GITCOMMIT)" && echo "PASS: cli version OK" || (echo "FAIL: cli version ($$ver) did not match" && exit 1)

ver="$$(/usr/libexec/docker/cli-plugins/docker-buildx docker-cli-plugin-metadata | awk '{ gsub(/[",:]/,"")}; $$1 == "Version" { print $$2 }')"; \
test "$$ver" = "$(BUILDX_VERSION)" && echo "PASS: docker-buildx version OK" || (echo "FAIL: docker-buildx version ($$ver) did not match" && exit 1)

ver="$$(/usr/libexec/docker/cli-plugins/docker-compose docker-cli-plugin-metadata | awk '{ gsub(/[",:]/,"")}; $$1 == "Version" { print $$2 }')"; \
test "$$ver" = "$(COMPOSE_VERSION)" && echo "PASS: docker-compose version OK" || (echo "FAIL: docker-compose version ($$ver) did not match" && exit 1)

Expand All @@ -67,20 +67,17 @@ override_dh_auto_install:
install -D -m 0644 cli/contrib/completion/fish/docker.fish debian/docker-ce-cli/usr/share/fish/vendor_completions.d/docker.fish
install -D -m 0644 cli/contrib/completion/zsh/_docker debian/docker-ce-cli/usr/share/zsh/vendor-completions/_docker
install -D -m 0755 cli/build/docker debian/docker-ce-cli/usr/bin/docker
set -e;cd /sources && \
tar xzf plugin-installers.tgz; \
for installer in plugins/*.installer; do \
DESTDIR=/root/build-deb/debian/docker-ce-cli \
PREFIX=/usr/libexec/docker/cli-plugins \
bash $${installer} install_plugin; \
done

# docker-ce install
install -D -m 0644 engine/contrib/init/systemd/docker.service debian/docker-ce/lib/systemd/system/docker.service
install -D -m 0644 engine/contrib/init/systemd/docker.socket debian/docker-ce/lib/systemd/system/docker.socket
install -D -m 0755 $(shell readlink -e engine/bundles/dynbinary-daemon/dockerd) debian/docker-ce/usr/bin/dockerd
install -D -m 0755 $(shell readlink -e engine/bundles/dynbinary-daemon/docker-proxy) debian/docker-ce/usr/bin/docker-proxy
install -D -m 0755 /usr/local/bin/docker-init debian/docker-ce/usr/bin/docker-init

# docker-buildx-plugin install
install -D -m 0755 /usr/libexec/docker/cli-plugins/docker-buildx debian/docker-buildx-plugin/usr/libexec/docker/cli-plugins/docker-buildx

# docker-compose-plugin install
install -D -m 0755 /usr/libexec/docker/cli-plugins/docker-compose debian/docker-compose-plugin/usr/libexec/docker/cli-plugins/docker-compose

Expand Down Expand Up @@ -110,6 +107,10 @@ override_dh_install:
dh_apparmor --profile-name=docker-ce -pdocker-ce

override_dh_gencontrol:
# Use separate version for the buildx-plugin package, then generate the other control files as usual
# TODO override "Source" field in control as well (to point to buildx, as it doesn't match the package name)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should probably (in a follow-up) duplicate the current deb/common directory for each of the packages we build, and split the control file accordingly. That will be a bit of work though (also splitting make targets etc etc)

Ideally, have separate stages in the Dockerfile to collect the source-files (and dependencies) for each, also to get out of the current situation where we bind-mount things and build packages in a docker run (which is horrible, because not only do we lose (e.g.) modules cache, but also currently requires some hacks chown / chmod to be done to adjust file permissions for cleaning up 😞

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah I already have a draft branch for that 👀

dh_gencontrol -pdocker-buildx-plugin -- -v$${BUILDX_VERSION#v}~$${DISTRO}-$${SUITE}

# Use separate version for the compose-plugin package, then generate the other control files as usual
# TODO override "Source" field in control as well (to point to compose, as it doesn't match the package name)
dh_gencontrol -pdocker-compose-plugin -- -v$${COMPOSE_VERSION#v}~$${DISTRO}-$${SUITE}
Expand Down
37 changes: 0 additions & 37 deletions plugins/.common
View file
Open in desktop

This file was deleted.

34 changes: 0 additions & 34 deletions plugins/buildx.installer
View file
Open in desktop

This file was deleted.

40 changes: 0 additions & 40 deletions plugins/scan.installer.disabled
View file
Open in desktop

This file was deleted.

24 changes: 14 additions & 10 deletions rpm/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ PLUGINS_DIR=$(realpath $(CURDIR)/../plugins)
GO_BASE_IMAGE=golang
GO_IMAGE?=$(GO_BASE_IMAGE):$(GO_VERSION)-buster
GEN_RPM_VER=$(shell ./gen-rpm-ver $(realpath $(CURDIR)/../src/github.com/docker/cli) "$(VERSION)")
GEN_BUILDX_RPM_VER=$(shell ./gen-rpm-ver $(realpath $(CURDIR)/../src/github.com/docker/buildx) "$(DOCKER_BUILDX_REF)")
GEN_COMPOSE_RPM_VER=$(shell ./gen-rpm-ver $(realpath $(CURDIR)/../src/github.com/docker/compose) "$(DOCKER_COMPOSE_REF)")
GEN_SCAN_RPM_VER=$(shell ./gen-rpm-ver $(realpath $(CURDIR)/../src/github.com/docker/scan-cli-plugin) "$(DOCKER_SCAN_REF)")
CLI_GITCOMMIT?=$(shell cd $(realpath $(CURDIR)/../src/github.com/docker/cli) && git rev-parse --short HEAD)
Expand All @@ -23,9 +24,9 @@ BUILD?=DOCKER_BUILDKIT=1 \


ifeq ($(ARCH),x86_64)
SPEC_FILES?=docker-ce.spec docker-ce-cli.spec docker-ce-rootless-extras.spec docker-compose-plugin.spec docker-scan-plugin.spec
SPEC_FILES?=docker-ce.spec docker-ce-cli.spec docker-ce-rootless-extras.spec docker-buildx-plugin.spec docker-compose-plugin.spec docker-scan-plugin.spec
else
SPEC_FILES?=docker-ce.spec docker-ce-cli.spec docker-ce-rootless-extras.spec docker-compose-plugin.spec
SPEC_FILES?=docker-ce.spec docker-ce-cli.spec docker-ce-rootless-extras.spec docker-buildx-plugin.spec docker-compose-plugin.spec
endif

SPECS?=$(addprefix SPECS/, $(SPEC_FILES))
Expand All @@ -35,6 +36,8 @@ RPMBUILD_FLAGS?=-ba\
--define '_release $(word 2,$(GEN_RPM_VER))' \
--define '_version $(word 1,$(GEN_RPM_VER))' \
--define '_origversion $(word 4, $(GEN_RPM_VER))' \
--define '_buildx_rpm_version $(word 1,$(GEN_BUILDX_RPM_VER))' \
--define '_buildx_version $(word 4,$(GEN_BUILDX_RPM_VER))' \
--define '_compose_rpm_version $(word 1,$(GEN_COMPOSE_RPM_VER))' \
--define '_compose_version $(word 4,$(GEN_COMPOSE_RPM_VER))' \
--define '_scan_rpm_version $(word 1,$(GEN_SCAN_RPM_VER))' \
Expand Down Expand Up @@ -98,7 +101,7 @@ $(DISTROS): sources
$(CHOWN) -R $(shell id -u):$(shell id -g) "rpmbuild/$@"

.PHONY: sources
sources: rpmbuild/SOURCES/engine.tgz rpmbuild/SOURCES/cli.tgz rpmbuild/SOURCES/compose.tgz rpmbuild/SOURCES/scan-cli-plugin.tgz rpmbuild/SOURCES/plugin-installers.tgz
sources: rpmbuild/SOURCES/engine.tgz rpmbuild/SOURCES/cli.tgz rpmbuild/SOURCES/buildx.tgz rpmbuild/SOURCES/compose.tgz rpmbuild/SOURCES/scan-cli-plugin.tgz

rpmbuild/SOURCES/engine.tgz:
mkdir -p $(@D)
Expand All @@ -116,6 +119,14 @@ rpmbuild/SOURCES/cli.tgz:
alpine \
tar -C / -c -z -f /v/cli.tgz --exclude .git cli

rpmbuild/SOURCES/buildx.tgz:
mkdir -p $(@D)
docker run --rm -w /v \
-v $(realpath $(CURDIR)/../src/github.com/docker/buildx):/buildx \
-v $(CURDIR)/$(@D):/v \
alpine \
tar -C / -c -z -f /v/buildx.tgz --exclude .git buildx

rpmbuild/SOURCES/compose.tgz:
mkdir -p $(@D)
docker run --rm -w /v \
Expand All @@ -132,13 +143,6 @@ rpmbuild/SOURCES/scan-cli-plugin.tgz:
alpine \
tar -C / -c -z -f /v/scan-cli-plugin.tgz --exclude .git scan-cli-plugin

rpmbuild/SOURCES/plugin-installers.tgz: $(wildcard ../plugins/*)
docker run --rm -w /v \
-v $(PLUGINS_DIR):/plugins \
-v $(CURDIR)/$(@D):/v \
alpine \
tar -C / -c -z -f /v/plugin-installers.tgz --exclude .git plugins

# See ARCHES in common.mk. Could not figure out how to match both distro and arch.
BUNDLES:=$(addsuffix .tar.gz,$(addprefix rpmbuild/bundles-ce-%-,$(ARCHES)))

Expand Down
Loading