From 0808de758fc60842149a636ad7c1b43e6b9727bb Mon Sep 17 00:00:00 2001 From: John Duimovich Date: Sat, 12 Feb 2022 07:44:50 -0500 Subject: [PATCH] cleanup build directory fix - use unique marker per run - avoid race condition when new pvs arrive in middle of run. - fix reference in doc to hard coded versions as they are stale - updated tests to use single "test" tag for now --- README.md | 6 ++--- tasks/appstudio-utils/README.md | 2 +- tasks/appstudio-utils/test-all-tasks.sh | 2 +- .../test-all-tasks/kustomization.yaml | 2 +- .../util-scripts/cleanup-build-directories.sh | 25 +++++++++++++------ 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ba9786901..7a77bae9f 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ This repository contains components that are installed or managed by the managed This includes default Pipelines and Tasks. You need to have bootstrapped a working appstudio configuration from (see `https://github.com/redhat-appstudio/infra-deployments`) for the dev of pipelines or new tasks. -Pipelines are delivered into App Studio via `quay.io/redhat-appstudio/build-templates-bundle:v0.1.3` (the tag will be updated every change) +Pipelines are delivered into App Studio via `quay.io/redhat-appstudio/build-templates-bundle:vX.Y.Z` (the tag will be updated every change) Tasks are delivered into App Studio via `quay.io/redhat-appstudio/appstudio-tasks`. Where each task is bundled and pushed into tag in format `${TASK_NAME}-${VERSION}` where version is the same as pipelines bundle version. -Currently a set of utilties are bundled with App Studio in `quay.io/redhat-appstudio/appstudio-utils:v0.1.3` as a convenience but tasks may be run from different per-task containers in future. +Currently a set of utilties are bundled with App Studio in `quay.io/redhat-appstudio/appstudio-utils:vX.Y.Z` as a convenience but tasks may be run from different per-task containers in future. ## Devmode @@ -30,7 +30,7 @@ Please test in _gitops mode_ when doing a new release into staging as it will be The tasks can be found in the `tasks` directories. Tasks are bundled and used by bundled pipelines. Tasks are not stored in the Cluster. For quick local innerloop style task development, you may install new Tasks in your local namespace manually and create your pipelines as well as the base task image to test new function. Tasks can be installed into local namespace using `oc apply -k tasks/appstudio-utils/util-tasks`. -There is a container which is used to support multiple set of tasks called `quay.io/redhat-appstudio/appstudio-utils:v0.1.3` , which is a single container which is used by multiple tasks. Tasks may also be in their own container as well however many simple tasks are utilities and will be packaged for app studio in a single container. Tasks can rely on other tasks in the system which are co-packed in a container allowing combined tasks (build-only vs build-deploy) which use the same core implementations. +There is a container which is used to support multiple set of tasks called `quay.io/redhat-appstudio/appstudio-utils:vX.Y.Z` , which is a single container which is used by multiple tasks. Tasks may also be in their own container as well however many simple tasks are utilities and will be packaged for app studio in a single container. Tasks can rely on other tasks in the system which are co-packed in a container allowing combined tasks (build-only vs build-deploy) which use the same core implementations. ## Release diff --git a/tasks/appstudio-utils/README.md b/tasks/appstudio-utils/README.md index 934354668..8dbbf3ee7 100644 --- a/tasks/appstudio-utils/README.md +++ b/tasks/appstudio-utils/README.md @@ -1,4 +1,4 @@ -# Source for quay.io/redhat-appstudioappstudio-utils:v0.1.3 +# Source for quay.io/redhat-appstudioappstudio-utils:vX.Y.Z This component provides an image which contains a suite of app-studio specific utilies. diff --git a/tasks/appstudio-utils/test-all-tasks.sh b/tasks/appstudio-utils/test-all-tasks.sh index 75b21894e..23315898a 100755 --- a/tasks/appstudio-utils/test-all-tasks.sh +++ b/tasks/appstudio-utils/test-all-tasks.sh @@ -2,7 +2,7 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" oc apply -k $SCRIPTDIR/test-all-tasks -tkn pipeline start test-all-tasks -w name=workspace,claimName=app-studio-default-workspace -p test-image=quay.io/redhat-appstudio/appstudio-utils:v0.1.3 --showlog --use-param-defaults +tkn pipeline start test-all-tasks -w name=workspace,claimName=app-studio-default-workspace -p test-image=quay.io/redhat-appstudio/appstudio-utils:test --showlog --use-param-defaults # cleanup manually, this is so you can inspect the results # if you delete immediately, the results are gone diff --git a/tasks/appstudio-utils/test-all-tasks/kustomization.yaml b/tasks/appstudio-utils/test-all-tasks/kustomization.yaml index 78d0dd17d..0a3abe5cc 100644 --- a/tasks/appstudio-utils/test-all-tasks/kustomization.yaml +++ b/tasks/appstudio-utils/test-all-tasks/kustomization.yaml @@ -11,4 +11,4 @@ patches: patch: |- - op: replace path: /spec/steps/0/image - value: quay.io/redhat-appstudio/appstudio-utils:v0.1.3 + value: quay.io/redhat-appstudio/appstudio-utils:test diff --git a/tasks/appstudio-utils/util-scripts/cleanup-build-directories.sh b/tasks/appstudio-utils/util-scripts/cleanup-build-directories.sh index 5119ccdb8..0257d2eb4 100755 --- a/tasks/appstudio-utils/util-scripts/cleanup-build-directories.sh +++ b/tasks/appstudio-utils/util-scripts/cleanup-build-directories.sh @@ -5,14 +5,24 @@ # rm unused marker in live pipeline-runs # rm all directories with unused marker # all other directories are live and have no marker (done) - -MARKER=.appstudio-mark-unused - +# if a directory is being gc'd by two copies of this task +# the marker will be unique so they can both run. +# live directories will be a noop +# dead directories can be deleted by either copy +# note, not recommended to run in parallel but will work. + +# use a unique marker per tasks based on time to NS +MARKER=.appstudio-mark-unused-$(date +"%Y%m%d%H%M%S%6N") +# Keep track of the existing pvs, any pvs added during this run +# will be ignored until a later run so that new pvs which were not +# originally marked dont get deleted due to not being marked. +CANDIDATES=./pv-* + cd workspace/source echo "Cleanup:" -BEFORE=$(du -h . | tail -n 1) +BEFORE=$(du -s) # mark all unused -for build in ./pv-* ; do +for build in $CANDIDATES ; do if [ -d "$build" ]; then echo "Directory: $build" echo "unused" > "$build/$MARKER" @@ -30,8 +40,9 @@ kubectl get pipelineruns --no-headers -o custom-columns=":metadata.name" | \ kubectl get pipelineruns --no-headers -o custom-columns=":metadata.name" | \ xargs -n 1 -I {} rm -f pv-{}/$MARKER 2> /dev/null + # if still marked unused, may be removed -for build in ./pv-* ; do +for build in $CANDIDATES ; do if [ -f "$build/$MARKER" ]; then echo "Removing: $(du -h $build | tail -n 1)" rm -rf $build @@ -41,7 +52,7 @@ for build in ./pv-* ; do fi fi done -AFTER=$(du -h . | tail -n 1) +AFTER=$(du -s) CMD=$(basename $0) if [ -d "$1" ]; then