Add all generation tools to Makefile

.github/workflows/test-go.yaml

@@ -35,7 +35,7 @@ jobs:
           git diff --cached --exit-code || (echo 'Please run "make manifests" to generate manifests' && exit 1);
       - name: Check auto-generated codes
         run: |
-          make generate && git add pkg/apis &&
+          make generate && git add pkg sdk &&
           git diff --cached --exit-code || (echo 'Please run "make generate" to generate Go codes' && exit 1);
       - name: Verify gofmt
         run: |

Makefile

@@ -40,8 +40,14 @@ help: ## Display this help.
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./pkg/apis/..." output:crd:artifacts:config=manifests/base/crds
 
-generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+generate: controller-gen ## Generate apidoc, sdk and code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. 
+	# Is controller gen used for code generation?
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./pkg/apis/..."
+	hack/update-codegen.sh
+	hack/python-sdk/gen-sdk.sh
+
+apidoc:
+	hack/generate-apidoc.sh
 
 fmt: ## Run go fmt against code.
 	go fmt ./...

docs/api/mpi_generated.asciidoc

@@ -60,7 +60,7 @@ Package v1 contains API Schema definitions for the kubeflow.org v1 API group
 | *`TypeMeta`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#typemeta-v1-meta[$$TypeMeta$$]__ | 
 | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#listmeta-v1-meta[$$ListMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`.
 
-| *`items`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-mpi-v1-mpijob[$$MPIJob$$]__ | 
+| *`items`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-mpi-v1-mpijob[$$MPIJob$$] array__ | 
 |===
 
 

docs/api/mxnet_generated.asciidoc

@@ -72,7 +72,7 @@ MXJobList contains a list of MXJob
 | *`TypeMeta`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#typemeta-v1-meta[$$TypeMeta$$]__ | 
 | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#listmeta-v1-meta[$$ListMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`.
 
-| *`items`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-mxnet-v1-mxjob[$$MXJob$$]__ | 
+| *`items`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-mxnet-v1-mxjob[$$MXJob$$] array__ | 
 |===
 
 
@@ -91,7 +91,7 @@ MXJobSpec defines the desired state of MXJob
 | Field | Description
 | *`runPolicy`* __xref:{anchor_prefix}-github-com-kubeflow-common-pkg-apis-common-v1-runpolicy[$$RunPolicy$$]__ | RunPolicy encapsulates various runtime policies of the distributed training job, for example how to clean up resources and how long the job can stay active.
 | *`jobMode`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-mxnet-v1-jobmodetype[$$JobModeType$$]__ | JobMode specify the kind of MXjob to do. Different mode may have different MXReplicaSpecs request
-| *`mxReplicaSpecs`* __object (keys:ReplicaType, values:ReplicaSpec)__ | MXReplicaSpecs is map of common.ReplicaType and common.ReplicaSpec specifies the MX replicas to run. For example,   {     "Scheduler": common.ReplicaSpec,     "Server": common.ReplicaSpec,     "Worker": common.ReplicaSpec,   }
+| *`mxReplicaSpecs`* __object (keys:ReplicaType, values:ReplicaSpec)__ | MXReplicaSpecs is map of common.ReplicaType and common.ReplicaSpec specifies the MX replicas to run. For example, { "Scheduler": common.ReplicaSpec, "Server": common.ReplicaSpec, "Worker": common.ReplicaSpec, }
 |===
 
 

docs/api/pytorch_generated.asciidoc

@@ -35,13 +35,17 @@ Package v1 contains API Schema definitions for the kubeflow.org v1 API group
 [cols="25a,75a", options="header"]
 |===
 | Field | Description
-| *`backend`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-pytorch-v1-rdzvbackend[$$RDZVBackend$$]__ | 
+| *`minReplicas`* __integer__ | minReplicas is the lower limit for the number of replicas to which the training job can scale down.  It defaults to null.
+| *`maxReplicas`* __integer__ | upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than MinReplicas, defaults to null.
+| *`rdzvBackend`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-pytorch-v1-rdzvbackend[$$RDZVBackend$$]__ | 
 | *`rdzvPort`* __integer__ | 
 | *`rdzvHost`* __string__ | 
 | *`rdzvId`* __string__ | 
 | *`rdzvConf`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-pytorch-v1-rdzvconf[$$RDZVConf$$] array__ | RDZVConf contains additional rendezvous configuration (<key1>=<value1>,<key2>=<value2>,...).
 | *`standalone`* __boolean__ | Start a local standalone rendezvous backend that is represented by a C10d TCP store on port 29400. Useful when launching single-node, multi-worker job. If specified --rdzv_backend, --rdzv_endpoint, --rdzv_id are auto-assigned; any explicitly set values are ignored.
 | *`nProcPerNode`* __integer__ | Number of workers per node; supported values: [auto, cpu, gpu, int].
+| *`maxRestarts`* __integer__ | 
+| *`metrics`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#metricspec-v2beta2-autoscaling[$$MetricSpec$$] array__ | Metrics contains the specifications which are used to calculate the desired replica count (the maximum replica count across all metrics will be used).  The desired replica count is calculated with multiplying the ratio between the target value and the current value by the current number of pods. Ergo, metrics used must decrease as the pod count is increased, and vice-versa.  See the individual metric source types for more information about how each type of metric must respond. If not set, the HPA will not be created.
 |===
 
 
@@ -83,7 +87,7 @@ PyTorchJobList is a list of PyTorchJobs.
 | *`TypeMeta`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#typemeta-v1-meta[$$TypeMeta$$]__ | Standard type metadata.
 | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#listmeta-v1-meta[$$ListMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`.
 
-| *`items`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-pytorch-v1-pytorchjob[$$PyTorchJob$$]__ | List of PyTorchJobs.
+| *`items`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-pytorch-v1-pytorchjob[$$PyTorchJob$$] array__ | List of PyTorchJobs.
 |===
 
 
@@ -102,7 +106,7 @@ PyTorchJobSpec is a desired state description of the PyTorchJob.
 | Field | Description
 | *`runPolicy`* __xref:{anchor_prefix}-github-com-kubeflow-common-pkg-apis-common-v1-runpolicy[$$RunPolicy$$]__ | RunPolicy encapsulates various runtime policies of the distributed training job, for example how to clean up resources and how long the job can stay active.
 | *`elasticPolicy`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-pytorch-v1-elasticpolicy[$$ElasticPolicy$$]__ | 
-| *`pytorchReplicaSpecs`* __object (keys:ReplicaType, values:ReplicaSpec)__ | A map of PyTorchReplicaType (type) to ReplicaSpec (value). Specifies the PyTorch cluster configuration. For example,   {     "Master": PyTorchReplicaSpec,     "Worker": PyTorchReplicaSpec,   }
+| *`pytorchReplicaSpecs`* __object (keys:ReplicaType, values:ReplicaSpec)__ | A map of PyTorchReplicaType (type) to ReplicaSpec (value). Specifies the PyTorch cluster configuration. For example, { "Master": PyTorchReplicaSpec, "Worker": PyTorchReplicaSpec, }
 |===
 
 

docs/api/tensorflow_generated.asciidoc

@@ -60,7 +60,7 @@ TFJobList is a list of TFJobs.
 | *`TypeMeta`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#typemeta-v1-meta[$$TypeMeta$$]__ | Standard type metadata.
 | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#listmeta-v1-meta[$$ListMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`.
 
-| *`items`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-tensorflow-v1-tfjob[$$TFJob$$]__ | List of TFJobs.
+| *`items`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-tensorflow-v1-tfjob[$$TFJob$$] array__ | List of TFJobs.
 |===
 
 
@@ -79,7 +79,7 @@ TFJobSpec is a desired state description of the TFJob.
 | Field | Description
 | *`runPolicy`* __xref:{anchor_prefix}-github-com-kubeflow-common-pkg-apis-common-v1-runpolicy[$$RunPolicy$$]__ | RunPolicy encapsulates various runtime policies of the distributed training job, for example how to clean up resources and how long the job can stay active.
 | *`successPolicy`* __SuccessPolicy__ | SuccessPolicy defines the policy to mark the TFJob as succeeded. Default to "", using the default rules.
-| *`tfReplicaSpecs`* __object (keys:ReplicaType, values:ReplicaSpec)__ | A map of TFReplicaType (type) to ReplicaSpec (value). Specifies the TF cluster configuration. For example,   {     "PS": ReplicaSpec,     "Worker": ReplicaSpec,   }
+| *`tfReplicaSpecs`* __object (keys:ReplicaType, values:ReplicaSpec)__ | A map of TFReplicaType (type) to ReplicaSpec (value). Specifies the TF cluster configuration. For example, { "PS": ReplicaSpec, "Worker": ReplicaSpec, }
 | *`enableDynamicWorker`* __boolean__ | A switch to enable dynamic worker
 |===
 

docs/api/xgboost_generated.asciidoc

@@ -60,7 +60,7 @@ XGBoostJobList contains a list of XGBoostJob
 | *`TypeMeta`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#typemeta-v1-meta[$$TypeMeta$$]__ | 
 | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#listmeta-v1-meta[$$ListMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`.
 
-| *`items`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-xgboost-v1-xgboostjob[$$XGBoostJob$$]__ | 
+| *`items`* __xref:{anchor_prefix}-github-com-kubeflow-training-operator-pkg-apis-xgboost-v1-xgboostjob[$$XGBoostJob$$] array__ | 
 |===
 
 

go.mod

@@ -3,13 +3,13 @@ module github.com/kubeflow/training-operator
 go 1.17
 
 require (
-	github.com/go-logr/logr v1.2.0
+	github.com/go-logr/logr v1.2.3
 	github.com/kubeflow/common v0.4.2
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.18.1
 	github.com/prometheus/client_golang v1.12.1
 	github.com/sirupsen/logrus v1.8.1
-	go.uber.org/zap v1.19.1
+	go.uber.org/zap v1.21.0
 	k8s.io/api v0.24.1
 	k8s.io/apimachinery v0.24.1
 	k8s.io/client-go v0.24.1
@@ -46,10 +46,10 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/go-cmp v0.5.5 // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
-	github.com/google/uuid v1.1.2 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
@@ -63,30 +63,30 @@ require (
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
+	golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
-	golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/tools v0.1.11 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/apiextensions-apiserver v0.24.0 // indirect
-	k8s.io/component-base v0.24.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0 // indirect
+	k8s.io/apiextensions-apiserver v0.24.1 // indirect
+	k8s.io/component-base v0.24.1 // indirect
 	k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect
 	k8s.io/klog/v2 v2.60.1 // indirect
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
-	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
+	sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 )