Skip to content

Commit

Permalink
Merge pull request #525 from ktock/ipfs-build
Browse files Browse the repository at this point in the history
Support builds using base images on IPFS (`nerdctl build --ipfs`)
  • Loading branch information
AkihiroSuda authored Nov 18, 2021
2 parents a81cb69 + 727bd43 commit 813dff2
Show file tree
Hide file tree
Showing 21 changed files with 905 additions and 12 deletions.
34 changes: 34 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,10 @@ It does not necessarily mean that the corresponding features are missing in cont
- [:whale: nerdctl compose build](#whale-nerdctl-compose-build)
- [:whale: nerdctl compose down](#whale-nerdctl-compose-down)
- [:whale: nerdctl compose ps](#whale-nerdctl-compose-ps)
- [IPFS management](#ipfs-management)
- [:nerd_face: nerdctl ipfs registry up](#nerd_face-nerdctl-ipfs-registry-up)
- [:nerd_face: nerdctl ipfs registry down](#nerd_face-nerdctl-ipfs-registry-down)
- [:nerd_face: nerdctl ipfs registry serve](#nerd_face-nerdctl-ipfs-registry-serve)
- [Global flags](#global-flags)
- [Unimplemented Docker commands](#unimplemented-docker-commands)
- [Additional documents](#additional-documents)
Expand Down Expand Up @@ -650,6 +654,7 @@ Flags:
- :whale: `--cache-from=CACHE`: External cache sources (eg. user/app:cache, type=local,src=path/to/dir) (compatible with `docker buildx build`)
- :whale: `--cache-to=CACHE`: Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir) (compatible with `docker buildx build`)
- :whale: `--platform=(amd64|arm64|...)`: Set target platform for build (compatible with `docker buildx build`)
- :nerd_face: `--ipfs`: Build image with pulling base images from IFPS. See [`./docs/ipfs.md`](./docs/ipfs.md) for details.

Unimplemented `docker build` flags: `--add-host`, `--iidfile`, `--label`, `--network`, `--squash`

Expand Down Expand Up @@ -1051,6 +1056,7 @@ Flags:
- :whale: `--no-color`: Produce monochrome output
- :whale: `--no-log-prefix`: Don't print prefix in logs
- :whale: `--build`: Build images before starting containers.
- :nerd_face: `--ipfs`: Build images with pulling base images from IFPS. See [`./docs/ipfs.md`](./docs/ipfs.md) for details.

Unimplemented `docker-compose up` (V1) flags: `--quiet-pull`, `--no-deps`, `--force-recreate`, `--always-recreate-deps`, `--no-recreate`,
`--no-start`, `--abort-on-container-exit`, `--attach-dependencies`, `--timeout`, `--renew-anon-volumes`, `--remove-orphans`, `--exit-code-from`,
Expand Down Expand Up @@ -1080,6 +1086,7 @@ Flags:
- :whale: `--build-arg`: Set build-time variables for services
- :whale: `--no-cache`: Do not use cache when building the image
- :whale: `--progress`: Set type of progress output (auto, plain, tty)
- :nerd_face: `--ipfs`: Build images with pulling base images from IFPS. See [`./docs/ipfs.md`](./docs/ipfs.md) for details.

Unimplemented `docker-compose build` (V1) flags: `--compress`, `--force-rm`, `--memory`, `--no-rm`, `--parallel`, `--pull`, `--quiet`

Expand All @@ -1102,6 +1109,33 @@ Unimplemented `docker-compose ps` (V1) flags: `--quiet`, `--services`, `--filter

Unimplemented `docker compose ps` (V2) flags: `--format`, `--status`

## IPFS management

### :nerd_face: nerdctl ipfs registry up
Start read-only local registry backed by IPFS.
See [`./docs/ipfs.md`](./docs/ipfs.md) for details.

Usage: `nerdctl ipfs registry up [OPTIONS]`

Flags:
- :nerd_face: `--listen-registry`: Address to listen (default `localhost:5050`)

### :nerd_face: nerdctl ipfs registry down
Stop and remove read-only local registry backed by IPFS.
See [`./docs/ipfs.md`](./docs/ipfs.md) for details.

Usage: `nerdctl ipfs registry down`

### :nerd_face: nerdctl ipfs registry serve
Serve read-only registry backed by IPFS on localhost.
Use `nerdctl ipfs registry up`.

Usage: `nerdctl ipfs registry serve [OPTIONS]`

Flags:
- :nerd_face: `--ipfs-address`: Multiaddr of IPFS API (default is pulled from `$IPFS_PATH/api` file. If `$IPFS_PATH` env var is not present, it defaults to `~/.ipfs`).
- :nerd_face: `--listen-registry`: Address to listen (default `localhost:5050`).

## Global flags
- :nerd_face: :blue_square: `--address`: containerd address, optionally with "unix://" prefix
- :nerd_face: :blue_square: `-a`, `--host`, `-H`: deprecated aliases of `--address`
Expand Down
16 changes: 16 additions & 0 deletions cmd/nerdctl/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ func newBuildCommand() *cobra.Command {
buildCommand.RegisterFlagCompletionFunc("platform", shellCompletePlatforms)
// #endregion

buildCommand.Flags().Bool("ipfs", false, "Allow pulling base images from IPFS")

return buildCommand
}

Expand All @@ -90,6 +92,20 @@ func buildAction(cmd *cobra.Command, args []string) error {
defer cleanup()
}

runIPFSRegistry, err := cmd.Flags().GetBool("ipfs")
if err != nil {
return err
}
if runIPFSRegistry {
logrus.Infof("Ensuring IPFS registry is running")
nerdctlCmd, nerdctlArgs := globalFlags(cmd)
if out, err := exec.Command(nerdctlCmd, append(nerdctlArgs, "ipfs", "registry", "up")...).CombinedOutput(); err != nil {
return fmt.Errorf("failed to start IPFS registry: %v: %v", string(out), err)
} else {
logrus.Infof("IPFS registry is running: %v", string(out))
}
}

quiet, err := cmd.Flags().GetBool("quiet")
if err != nil {
return err
Expand Down
26 changes: 26 additions & 0 deletions cmd/nerdctl/build_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,32 @@ CMD ["echo", "nerdctl-build-test-string"]
base.Cmd("run", "--rm", imageName).AssertOutContains("nerdctl-build-test-string")
}

func TestIPFSBuild(t *testing.T) {
testutil.DockerIncompatible(t)
requiresIPFS(t)
testutil.RequiresBuild(t)
base := testutil.NewBase(t)
ipfsCID := pushImageToIPFS(t, base, testutil.AlpineImage)
ipfsCIDBase := strings.TrimPrefix(ipfsCID, "ipfs://")

const imageName = "nerdctl-build-test"
defer base.Cmd("rmi", imageName).Run()

dockerfile := fmt.Sprintf(`FROM localhost:5050/ipfs/%s
CMD ["echo", "nerdctl-build-test-string"]
`, ipfsCIDBase)

buildCtx, err := createBuildContext(dockerfile)
assert.NilError(t, err)
defer os.RemoveAll(buildCtx)

defer base.Cmd("ipfs", "registry", "down").AssertOK()
base.Cmd("build", "--ipfs", "-t", imageName, buildCtx).AssertOK()
base.Cmd("build", buildCtx, "--ipfs", "-t", imageName).AssertOK()

base.Cmd("run", "--rm", imageName).AssertOutContains("nerdctl-build-test-string")
}

func TestBuildFromStdin(t *testing.T) {
testutil.RequiresBuild(t)
base := testutil.NewBase(t)
Expand Down
8 changes: 8 additions & 0 deletions cmd/nerdctl/compose_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ func newComposeBuildCommand() *cobra.Command {
composeBuildCommand.Flags().StringArray("build-arg", nil, "Set build-time variables for services.")
composeBuildCommand.Flags().Bool("no-cache", false, "Do not use cache when building the image.")
composeBuildCommand.Flags().String("progress", "", "Set type of progress output")

composeBuildCommand.Flags().Bool("ipfs", false, "Allow pulling base images from IPFS during build")

return composeBuildCommand
}

Expand All @@ -54,6 +57,10 @@ func composeBuildAction(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
enableIPFS, err := cmd.Flags().GetBool("ipfs")
if err != nil {
return err
}

client, ctx, cancel, err := newClient(cmd)
if err != nil {
Expand All @@ -69,6 +76,7 @@ func composeBuildAction(cmd *cobra.Command, args []string) error {
Args: buildArg,
NoCache: noCache,
Progress: progress,
IPFS: enableIPFS,
}
return c.Build(ctx, bo)
}
6 changes: 6 additions & 0 deletions cmd/nerdctl/compose_up.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func newComposeUpCommand() *cobra.Command {
composeUpCommand.Flags().Bool("no-color", false, "Produce monochrome output")
composeUpCommand.Flags().Bool("no-log-prefix", false, "Don't print prefix in logs")
composeUpCommand.Flags().Bool("build", false, "Build images before starting containers.")
composeUpCommand.Flags().Bool("ipfs", false, "Allow pulling base images from IPFS during build")
return composeUpCommand
}

Expand All @@ -53,6 +54,10 @@ func composeUpAction(cmd *cobra.Command, services []string) error {
if err != nil {
return err
}
enableIPFS, err := cmd.Flags().GetBool("ipfs")
if err != nil {
return err
}

client, ctx, cancel, err := newClient(cmd)
if err != nil {
Expand All @@ -69,6 +74,7 @@ func composeUpAction(cmd *cobra.Command, services []string) error {
NoColor: noColor,
NoLogPrefix: noLogPrefix,
ForceBuild: build,
IPFS: enableIPFS,
}
return c.Up(ctx, uo, services)
}
38 changes: 38 additions & 0 deletions cmd/nerdctl/compose_up_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,44 @@ COPY index.html /usr/share/nginx/html/index.html
assert.Assert(t, strings.Contains(string(respBody), indexHTML))
}

func TestIPFSComposeUpBuild(t *testing.T) {
requiresIPFS(t)
testutil.DockerIncompatible(t)
testutil.RequiresBuild(t)
base := testutil.NewBase(t)
ipfsCID := pushImageToIPFS(t, base, testutil.NginxAlpineImage)
ipfsCIDBase := strings.TrimPrefix(ipfsCID, "ipfs://")

const dockerComposeYAML = `
services:
web:
build: .
ports:
- 8080:80
`
dockerfile := fmt.Sprintf(`FROM localhost:5050/ipfs/%s
COPY index.html /usr/share/nginx/html/index.html
`, ipfsCIDBase)
indexHTML := t.Name()

comp := testutil.NewComposeDir(t, dockerComposeYAML)
defer comp.CleanUp()

comp.WriteFile("Dockerfile", dockerfile)
comp.WriteFile("index.html", indexHTML)

base.ComposeCmd("-f", comp.YAMLFullPath(), "up", "-d", "--build", "--ipfs").AssertOK()
defer base.Cmd("ipfs", "registry", "down").AssertOK()
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

resp, err := httpGet("http://127.0.0.1:8080", 50)
assert.NilError(t, err)
respBody, err := io.ReadAll(resp.Body)
assert.NilError(t, err)
t.Logf("respBody=%q", respBody)
assert.Assert(t, strings.Contains(string(respBody), indexHTML))
}

func TestComposeUpMultiNet(t *testing.T) {
base := testutil.NewBase(t)

Expand Down
36 changes: 36 additions & 0 deletions cmd/nerdctl/ipfs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"github.com/spf13/cobra"
)

func newIPFSCommand() *cobra.Command {
cmd := &cobra.Command{
Annotations: map[string]string{Category: Management},
Use: "ipfs",
Short: "Distributing images on IPFS",
RunE: unknownSubcommandAction,
SilenceUsage: true,
SilenceErrors: true,
}
cmd.AddCommand(
newIPFSRegistryCommand(),
)
return cmd
}
38 changes: 38 additions & 0 deletions cmd/nerdctl/ipfs_registry.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"github.com/spf13/cobra"
)

func newIPFSRegistryCommand() *cobra.Command {
cmd := &cobra.Command{
Annotations: map[string]string{Category: Management},
Use: "registry",
Short: "Manage read-only registry backed by IPFS",
RunE: unknownSubcommandAction,
SilenceUsage: true,
SilenceErrors: true,
}
cmd.AddCommand(
newIPFSRegistryServeCommand(),
newIPFSRegistryUpCommand(),
newIPFSRegistryDownCommand(),
)
return cmd
}
67 changes: 67 additions & 0 deletions cmd/nerdctl/ipfs_registry_down.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"context"
"fmt"
"os/exec"

"github.com/containerd/nerdctl/pkg/idutil/containerwalker"
"github.com/spf13/cobra"
)

func newIPFSRegistryDownCommand() *cobra.Command {
var ipfsRegistryDownCommand = &cobra.Command{
Use: "down",
Short: "stop registry as a background container \"ipfs-registry\".",
RunE: ipfsRegistryDownAction,
SilenceUsage: true,
SilenceErrors: true,
}

return ipfsRegistryDownCommand
}

func ipfsRegistryDownAction(cmd *cobra.Command, args []string) error {
client, ctx, cancel, err := newClient(cmd)
if err != nil {
return err
}
defer cancel()
walker := &containerwalker.ContainerWalker{
Client: client,
OnFound: func(ctx context.Context, found containerwalker.Found) error {
return nil
},
}
nc, err := walker.Walk(ctx, ipfsRegistryContainerName)
if err != nil {
return err
}
if nc == 0 {
return fmt.Errorf("ipfs registry %q doesn't exist", ipfsRegistryContainerName)
}
nerdctlCmd, nerdctlArgs := globalFlags(cmd)
if out, err := exec.Command(nerdctlCmd, append(nerdctlArgs, "stop", ipfsRegistryContainerName)...).CombinedOutput(); err != nil {
return fmt.Errorf("failed to stop registry: %v: %v", string(out), err)
}
if out, err := exec.Command(nerdctlCmd, append(nerdctlArgs, "rm", ipfsRegistryContainerName)...).CombinedOutput(); err != nil {
return fmt.Errorf("failed to remove registry: %v: %v", string(out), err)
}
return nil
}
Loading

0 comments on commit 813dff2

Please sign in to comment.