diff --git a/.drone-local.yml b/.drone-local.yml index f3db0517..a7e69353 100644 --- a/.drone-local.yml +++ b/.drone-local.yml @@ -12,12 +12,6 @@ clone: depth: 50 steps: - # This is needed for the tags. And the tags are needed to determine version. - - name: fetch - image: docker:git - commands: - - git fetch --tags -f - - name: configure-buckets image: minio/mc:RELEASE.2020-10-03T02-54-56Z commands: @@ -37,7 +31,7 @@ steps: CGO_ENABLED: 0 - name: lint - image: golang:1.18.4 + image: docker.io/golangci/golangci-lint:v1.46.2 commands: - make lint environment: @@ -237,7 +231,7 @@ steps: services: - name: minio - image: minio/minio:RELEASE.2020-11-06T23-17-07Z + image: minio/minio:RELEASE.2022-07-15T03-44-22Z commands: - minio server /data environment: @@ -247,7 +241,7 @@ services: ports: - 9000 - name: fakegcs - image: fsouza/fake-gcs-server:1.18.3 + image: fsouza/fake-gcs-server:1.38.3 ports: - 4443 commands: @@ -259,7 +253,7 @@ services: commands: - /entrypoint foo:pass:::sftp_test bar:pass:::plugin_test - name: azurite - image: mcr.microsoft.com/azure-storage/azurite:3.10.0 + image: mcr.microsoft.com/azure-storage/azurite:3.18.0 commands: - azurite-blob --blobHost 0.0.0.0 ports: diff --git a/CHANGELOG.md b/CHANGELOG.md index c2497d91..78d76a95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- [#209](https://github.com/meltwater/drone-cache/pull/209) Added double star directory searching in mounts (e.g. `path/**/subdir`) - [#198](https://github.com/meltwater/drone-cache/pull/198) Add `hashFiles` template function to generate the SHA256 hash of multiple files ### Changed diff --git a/Dockerfile b/Dockerfile index 9d502063..0a26c34b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ # This file is designed to only used by goreleaser. FROM golang:1.18.4 AS builder -RUN apk add --update --no-cache ca-certificates tzdata && update-ca-certificates + +RUN update-ca-certificates FROM scratch as runner diff --git a/docs/examples/drone.md b/docs/examples/drone.md index 696c81ad..17cdb647 100644 --- a/docs/examples/drone.md +++ b/docs/examples/drone.md @@ -222,3 +222,40 @@ steps: rebuild: true debug: true ``` + +### Glob Double Star Mounting + +```yaml +kind: pipeline +name: default + +steps: + - name: restore-cache-debug + image: meltwater/drone-cache + settings: + pull: true + restore: true + debug: true + mount: + - "node_modules" + - "packages/**/dist" + - "packages/**/node_modules" + + - name: build + image: npm + pull: true + commands: + - npm build + + + - name: restore-cache-debug + image: meltwater/drone-cache + settings: + pull: true + rebuild: true + debug: true + mount: + - "node_modules" + - "packages/**/dist" + - "packages/**/node_modules" +``` diff --git a/go.mod b/go.mod index 1b2c85ed..7032242f 100644 --- a/go.mod +++ b/go.mod @@ -3,18 +3,19 @@ module github.com/meltwater/drone-cache go 1.18 require ( - cloud.google.com/go/storage v1.23.0 + cloud.google.com/go/storage v1.24.0 github.com/Azure/azure-storage-blob-go v0.15.0 - github.com/aws/aws-sdk-go v1.44.55 + github.com/aws/aws-sdk-go v1.44.60 + github.com/bmatcuk/doublestar/v4 v4.2.0 github.com/dustin/go-humanize v1.0.0 github.com/go-kit/log v0.2.1 github.com/google/go-cmp v0.5.8 - github.com/klauspost/compress v1.15.8 + github.com/klauspost/compress v1.15.9 github.com/pkg/sftp v1.13.5 - github.com/urfave/cli/v2 v2.11.0 - golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d - golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 - google.golang.org/api v0.87.0 + github.com/urfave/cli/v2 v2.11.1 + golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa + golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c + google.golang.org/api v0.88.0 ) require ( @@ -30,7 +31,6 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect github.com/googleapis/gax-go/v2 v2.4.0 // indirect - github.com/googleapis/go-type-adapters v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/mattn/go-ieproxy v0.0.7 // indirect @@ -38,12 +38,12 @@ require ( github.com/stretchr/testify v1.7.1 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/net v0.0.0-20220708220712-1185a9018129 // indirect - golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect + golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect + golang.org/x/sys v0.0.0-20220721230656-c6bc011c0c49 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220714211235-042d03aeabc9 // indirect + google.golang.org/genproto v0.0.0-20220720214146-176da50484ac // indirect google.golang.org/grpc v1.48.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 7f68ebd1..daf59a17 100644 --- a/go.sum +++ b/go.sum @@ -58,8 +58,9 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= -cloud.google.com/go/storage v1.23.0 h1:wWRIaDURQA8xxHguFCshYepGlrWIrbBnAmc7wfg07qY= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.24.0 h1:a4N0gIkx83uoVFGz8B2eAV3OhN90QoWF5OZWLKl39ig= +cloud.google.com/go/storage v1.24.0/go.mod h1:3xrJEFMXBsQLgxwThyjuD3aYlroL0TMRec1ypGUQ0KE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= @@ -80,8 +81,10 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/aws/aws-sdk-go v1.44.55 h1:h+p61sPEsLOpnQ2mKnGPrIe1MFUKwwA0X5eQYAcjOMU= -github.com/aws/aws-sdk-go v1.44.55/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.60 h1:KTTogelVR+4dWiIPl7eyxoxaJkziChON6/Y/hVfTipk= +github.com/aws/aws-sdk-go v1.44.60/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/bmatcuk/doublestar/v4 v4.2.0 h1:Qu+u9wR3Vd89LnlLMHvnZ5coJMWKQamqdz9/p5GNthA= +github.com/bmatcuk/doublestar/v4 v4.2.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -212,7 +215,6 @@ github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/Oth github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -226,8 +228,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA= -github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -261,8 +263,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/urfave/cli/v2 v2.11.0 h1:c6bD90aLd2iEsokxhxkY5Er0zA2V9fId2aJfwmrF+do= -github.com/urfave/cli/v2 v2.11.0/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhAyPlwvo= +github.com/urfave/cli/v2 v2.11.1 h1:UKK6SP7fV3eKOefbS87iT9YHefv7iB/53ih6e+GNAsE= +github.com/urfave/cli/v2 v2.11.1/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhAyPlwvo= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -288,8 +290,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -373,8 +375,8 @@ golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0= -golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -396,8 +398,8 @@ golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 h1:VnGaRqoLmqZH/3TMLJwYCEWkR4j1nuIU1U9TvbqsDUw= -golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c h1:q3gFqPqH7NVofKo3c3yETAP//pPI+G5mvB7qqj1Y5kY= +golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -474,8 +476,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220721230656-c6bc011c0c49 h1:TMjZDarEwf621XDryfitp/8awEhiZNiwgphKlTMGRIg= +golang.org/x/sys v0.0.0-20220721230656-c6bc011c0c49/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -593,8 +595,8 @@ google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3 google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= -google.golang.org/api v0.87.0 h1:pUQVF/F+X7Tl1lo4LJoJf5BOpjtmINU80p9XpYTU2p4= -google.golang.org/api v0.87.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.88.0 h1:MPwxQRqpyskYhr2iNyfsQ8R06eeyhe7UEuR30p136ZQ= +google.golang.org/api v0.88.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -684,8 +686,8 @@ google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljW google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220714211235-042d03aeabc9 h1:zfXhTgBfGlIh3jMXN06W8qbhFGsh6MJNJiYEuhTddOI= -google.golang.org/genproto v0.0.0-20220714211235-042d03aeabc9/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220720214146-176da50484ac h1:EOa+Yrhx1C0O+4pHeXeWrCwdI0tWI6IfUU56Vebs9wQ= +google.golang.org/genproto v0.0.0-20220720214146-176da50484ac/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/internal/plugin/config.go b/internal/plugin/config.go index ce2536de..5c295c0a 100644 --- a/internal/plugin/config.go +++ b/internal/plugin/config.go @@ -1,8 +1,12 @@ package plugin import ( + "fmt" + "io/fs" + "strings" "time" + "github.com/bmatcuk/doublestar/v4" "github.com/meltwater/drone-cache/storage/backend/azure" "github.com/meltwater/drone-cache/storage/backend/filesystem" "github.com/meltwater/drone-cache/storage/backend/gcs" @@ -38,3 +42,27 @@ type Config struct { Azure azure.Config GCS gcs.Config } + +// HandleMount runs prior to Rebuild and Restoring of caches to handle unique +// paths such as double-star globs. +func (c *Config) HandleMount(fsys fs.FS) error { + mountLen := len(c.Mount) + if mountLen > 0 { + for i, mount := range c.Mount { + if strings.Contains(mount, "**") { + // Remove the glob from the original mount list + c.Mount[i] = c.Mount[mountLen-1] + c.Mount = c.Mount[:mountLen-1] + + globMounts, err := doublestar.Glob(fsys, mount) + if err != nil { + return fmt.Errorf("glob handle mount error <%s>, %w", mount, err) + } + + c.Mount = append(c.Mount, globMounts...) + } + } + } + + return nil +} diff --git a/internal/plugin/config_test.go b/internal/plugin/config_test.go new file mode 100644 index 00000000..6a283d77 --- /dev/null +++ b/internal/plugin/config_test.go @@ -0,0 +1,73 @@ +package plugin + +import ( + "fmt" + "os" + "reflect" + "testing" + + "github.com/meltwater/drone-cache/test" +) + +const ( + testRootGlob = "testglobdata" +) + +func TestHandleMount(t *testing.T) { + test.Ok(t, os.Mkdir(testRootGlob, 0755)) + t.Cleanup(func() { + os.RemoveAll(testRootGlob) + }) + cases := []struct { + name string + mounts []string + expectedMounts []string + makeFiles func() + }{ + { + name: "handle-mount-single", + mounts: []string{"test/single"}, + expectedMounts: []string{"test/single"}, + makeFiles: func() {}, + }, + { + name: "handle-mount-nested", + mounts: []string{"test/a", "test/b"}, + expectedMounts: []string{"test/a", "test/b"}, + makeFiles: func() {}, + }, + { + name: "handle-mount-glob-empty", + mounts: []string{"test/**", "test/b"}, + expectedMounts: []string{"test/b"}, + makeFiles: func() {}, + }, + { + name: "handle-mount-glob-notempty", + mounts: []string{fmt.Sprintf("%s/%s", testRootGlob, "test/**/test")}, + expectedMounts: []string{ + fmt.Sprintf("%s/%s", testRootGlob, "test/nestedA/test"), + fmt.Sprintf("%s/%s", testRootGlob, "test/nestedB/test"), + }, + makeFiles: func() { + // Make test directories for glob to work properly + os.MkdirAll(fmt.Sprintf("%s/%s", testRootGlob, "test/nestedA/test"), 0755) + os.MkdirAll(fmt.Sprintf("%s/%s", testRootGlob, "test/nestedB/test"), 0755) + }, + }, + } + + for _, tc := range cases { + c := Config{} + c.Mount = tc.mounts + + tc.makeFiles() + cwd, err := os.Getwd() + test.Ok(t, err) + fsys := os.DirFS(cwd) + test.Ok(t, c.HandleMount(fsys)) + + test.Assert(t, reflect.DeepEqual(c.Mount, tc.expectedMounts), + "expected mount differs from handled mount result:\nexpected: %v\ngot:%v", tc.expectedMounts, c.Mount) + } +} diff --git a/internal/plugin/plugin.go b/internal/plugin/plugin.go index cf9919ee..c2920c6d 100644 --- a/internal/plugin/plugin.go +++ b/internal/plugin/plugin.go @@ -122,7 +122,19 @@ func (p *Plugin) Exec() error { // nolint: funlen,cyclop options..., ) - // 4. Select mode + // 4. Glob match mounts if doublestar paths exist + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("get working directory, %w", err) + } + + fsys := os.DirFS(cwd) + + if err = p.Config.HandleMount(fsys); err != nil { + return fmt.Errorf("exec handle mount call, %w", err) + } + + // 5. Select mode if cfg.Rebuild { if err := c.Rebuild(p.Config.Mount); err != nil { level.Debug(p.logger).Log("err", fmt.Sprintf("%+v\n", err)) diff --git a/internal/plugin/plugin_test.go b/internal/plugin/plugin_test.go index e59ede57..0c56b8c8 100644 --- a/internal/plugin/plugin_test.go +++ b/internal/plugin/plugin_test.go @@ -107,6 +107,13 @@ func TestPlugin(t *testing.T) { }, success: true, }, + { + name: "existing-mount-with-glob-files", + mount: func(name string) []string { + return exampleNestedFileTreeWithGlob(t, name, make([]byte, 1*1024)) + }, + success: true, + }, { name: "existing-mount-with-cache-key", mount: func(name string) []string { @@ -142,7 +149,12 @@ func TestPlugin(t *testing.T) { c := defaultConfig() setup(t, c, name) paths := tc.mount(tc.name) - mount(c, paths...) + c = mount(c, paths...) + cwd, err := os.Getwd() + test.Ok(t, err) + fsys := os.DirFS(cwd) + test.Ok(t, c.HandleMount(fsys)) + cacheKey(c, tc.cacheKey) format(c, f) @@ -161,7 +173,7 @@ func TestPlugin(t *testing.T) { restoreRoot, cleanup := test.CreateTempDir(t, sanitize(name), testRootMoved) t.Cleanup(cleanup) - for _, p := range paths { + for _, p := range c.Mount { rel, err := filepath.Rel(testRootMounted, p) test.Ok(t, err) dst := filepath.Join(restoreRoot, rel) @@ -182,7 +194,7 @@ func TestPlugin(t *testing.T) { } // Compare - test.EqualDirs(t, restoreRoot, testRootMounted, paths) + test.EqualDirs(t, restoreRoot, testRootMounted, c.Mount) }) } } @@ -213,6 +225,7 @@ func restore(c *Config) *Config { func mount(c *Config, mount ...string) *Config { c.Mount = mount + return c } @@ -312,6 +325,33 @@ func exampleFileTreeWithSymlinks(t *testing.T, name string, content []byte) []st return []string{file, dir, symDir} } +func exampleNestedFileTreeWithGlob(t *testing.T, name string, content []byte) []string { + name = sanitize(name) + + dirA, dirAClean := test.CreateTempDir(t, name, testRootMounted) + t.Cleanup(dirAClean) + + nestedDirA := fmt.Sprintf("%s/test", dirA) + os.Mkdir(nestedDirA, 0o755) + t.Cleanup(func() { os.RemoveAll(nestedDirA) }) + + _, nestedFilesAClean := test.CreateTempFilesInDir(t, name, content, nestedDirA) + t.Cleanup(nestedFilesAClean) + + dirB, dirBClean := test.CreateTempDir(t, name, testRootMounted) + t.Cleanup(dirBClean) + + nestedDirB := fmt.Sprintf("%s/test", dirB) + os.Mkdir(nestedDirB, 0o755) + t.Cleanup(func() { os.RemoveAll(nestedDirB) }) + + _, nestedFilesBClean := test.CreateTempFilesInDir(t, name, content, nestedDirB) + t.Cleanup(nestedFilesBClean) + + globPath := fmt.Sprintf("%s/**/test", testRootMounted) + return []string{globPath} +} + // Setup func setupAzure(t *testing.T, c *Config, name string) {