Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support specifying an initialize sql file on TiDB' s first bootstrap #4862

Merged
merged 11 commits into from
Feb 16, 2023
15 changes: 15 additions & 0 deletions docs/api-references/docs.md
Original file line number Diff line number Diff line change
Expand Up @@ -16587,6 +16587,21 @@ TiDBInitializer
<p>Initializer is the init configurations of TiDB</p>
</td>
</tr>
<tr>
<td>
<code>bootstrapSQLConfigMapName</code></br>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>BootstrapSQLConfigMapName is the name of the ConfigMap which contains the bootstrap SQL file with the key <code>bootstrap-sql</code>,
which will only be executed when a TiDB cluster bootstrap on the first time.
The field should be set ONLY when create a TC, since it only take effect on the first time bootstrap.
Only v6.6.0+ supports this feature.</p>
</td>
</tr>
</tbody>
</table>
<h3 id="tidbstatus">TiDBStatus</h3>
Expand Down
2 changes: 2 additions & 0 deletions manifests/crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24698,6 +24698,8 @@ spec:
type: string
binlogEnabled:
type: boolean
bootstrapSQLConfigMapName:
type: string
config:
x-kubernetes-preserve-unknown-fields: true
configUpdateStrategy:
Expand Down
2 changes: 2 additions & 0 deletions manifests/crd/v1/pingcap.com_tidbclusters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11443,6 +11443,8 @@ spec:
type: string
binlogEnabled:
type: boolean
bootstrapSQLConfigMapName:
type: string
config:
x-kubernetes-preserve-unknown-fields: true
configUpdateStrategy:
Expand Down
2 changes: 2 additions & 0 deletions manifests/crd/v1beta1/pingcap.com_tidbclusters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11428,6 +11428,8 @@ spec:
type: string
binlogEnabled:
type: boolean
bootstrapSQLConfigMapName:
type: string
config:
x-kubernetes-preserve-unknown-fields: true
configUpdateStrategy:
Expand Down
2 changes: 2 additions & 0 deletions manifests/crd_v1beta1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24678,6 +24678,8 @@ spec:
type: string
binlogEnabled:
type: boolean
bootstrapSQLConfigMapName:
type: string
config:
x-kubernetes-preserve-unknown-fields: true
configUpdateStrategy:
Expand Down
7 changes: 7 additions & 0 deletions pkg/apis/pingcap/v1alpha1/openapi_generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 8 additions & 0 deletions pkg/apis/pingcap/v1alpha1/tidbcluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -975,6 +975,14 @@ func (tc *TidbCluster) IsTiDBBinlogEnabled() bool {
return *binlogEnabled
}

func (tidb *TiDBSpec) IsBootstrapSQLEnabled() bool {
fgksgf marked this conversation as resolved.
Show resolved Hide resolved
if tidb.BootstrapSQLConfigMapName != nil && *tidb.BootstrapSQLConfigMapName != "" {
return true
}

return false
}

func (tidb *TiDBSpec) IsTLSClientEnabled() bool {
return tidb.TLSClient != nil && tidb.TLSClient.Enabled
}
Expand Down
7 changes: 7 additions & 0 deletions pkg/apis/pingcap/v1alpha1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -914,6 +914,13 @@ type TiDBSpec struct {
//
// +optional
Initializer *TiDBInitializer `json:"initializer,omitempty"`

// BootstrapSQLConfigMapName is the name of the ConfigMap which contains the bootstrap SQL file with the key `bootstrap-sql`,
// which will only be executed when a TiDB cluster bootstrap on the first time.
// The field should be set ONLY when create a TC, since it only take effect on the first time bootstrap.
// Only v6.6.0+ supports this feature.
// +optional
BootstrapSQLConfigMapName *string `json:"bootstrapSQLConfigMapName,omitempty"`
}

type TiDBInitializer struct {
Expand Down
18 changes: 18 additions & 0 deletions pkg/apis/pingcap/v1alpha1/validation/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -536,6 +536,7 @@ func ValidateUpdateTidbCluster(old, tc *v1alpha1.TidbCluster) field.ErrorList {
"The instance must not be mutate or set value other than the cluster name"))
}
allErrs = append(allErrs, validateUpdatePDConfig(old.Spec.PD, tc.Spec.PD, field.NewPath("spec.pd.config"))...)
allErrs = append(allErrs, disallowMutateBootstrapSQLConfigMapName(old.Spec.TiDB, tc.Spec.TiDB, field.NewPath("spec.tidb.bootstrapSQLConfigMapName"))...)
allErrs = append(allErrs, disallowUsingLegacyAPIInNewCluster(old, tc)...)

return allErrs
Expand Down Expand Up @@ -643,6 +644,23 @@ func validateUpdatePDConfig(oldPdSpec, pdSpec *v1alpha1.PDSpec, path *field.Path
return allErrs
}

// disallowMutateBootstrapSQLConfigMapName checks if user mutate the bootstrapSQLConfigMapName field.
// Only allow to update bootstrapSQLConfigMapName from non-nil to nil.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do we need to support updates from non-nil to nil?

Copy link
Contributor Author

@fgksgf fgksgf Feb 13, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the user deletes the bootstrap sql configmap and also wants to remove it from tidb pods.

func disallowMutateBootstrapSQLConfigMapName(old, new *v1alpha1.TiDBSpec, p *field.Path) field.ErrorList {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it seems there are no need to restrict this

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because it only take effect at the first bootstrap, so updating the field makes no sense but make tidb restart.

allErrs := field.ErrorList{}
if old == nil || new == nil {
return allErrs
}

bootstrapSQLSpecified := old.BootstrapSQLConfigMapName != nil && new.BootstrapSQLConfigMapName != nil
if (bootstrapSQLSpecified && *old.BootstrapSQLConfigMapName != *new.BootstrapSQLConfigMapName) ||
(!bootstrapSQLSpecified && new.BootstrapSQLConfigMapName != nil) {
return append(allErrs, field.Invalid(p, new.BootstrapSQLConfigMapName, "bootstrapSQLConfigMapName is immutable"))
}

return allErrs
}

func validateDeleteSlots(annotations map[string]string, key string, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if annotations != nil {
Expand Down
71 changes: 71 additions & 0 deletions pkg/apis/pingcap/v1alpha1/validation/validation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -749,3 +749,74 @@ func TestValidatePDSpec(t *testing.T) {
})
}
}

func Test_disallowMutateBootstrapSQLConfigMapName(t *testing.T) {
g := NewGomegaWithT(t)
tests := []struct {
name string
old *v1alpha1.TiDBSpec
new *v1alpha1.TiDBSpec
wantError bool
}{
{
name: "no change, both nil",
old: &v1alpha1.TiDBSpec{
BootstrapSQLConfigMapName: nil,
},
new: &v1alpha1.TiDBSpec{
BootstrapSQLConfigMapName: nil,
},
wantError: false,
},
{
name: "no change, both non-nil",
old: &v1alpha1.TiDBSpec{
BootstrapSQLConfigMapName: pointer.StringPtr("old"),
},
new: &v1alpha1.TiDBSpec{
BootstrapSQLConfigMapName: pointer.StringPtr("old"),
},
wantError: false,
},
{
name: "mutate from non-nil to nil",
old: &v1alpha1.TiDBSpec{
BootstrapSQLConfigMapName: pointer.StringPtr("old"),
},
new: &v1alpha1.TiDBSpec{
BootstrapSQLConfigMapName: nil,
},
wantError: false,
},
{
name: "mutate from nil to non-nil",
old: &v1alpha1.TiDBSpec{
BootstrapSQLConfigMapName: nil,
},
new: &v1alpha1.TiDBSpec{
BootstrapSQLConfigMapName: pointer.StringPtr("new"),
},
wantError: true,
},
{
name: "mutate from non-nil to non-nil",
old: &v1alpha1.TiDBSpec{
BootstrapSQLConfigMapName: pointer.StringPtr("old"),
},
new: &v1alpha1.TiDBSpec{
BootstrapSQLConfigMapName: pointer.StringPtr("new"),
},
wantError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
errs := disallowMutateBootstrapSQLConfigMapName(tt.old, tt.new, field.NewPath("spec.tidb.bootstrapSQLConfigMapName"))
if tt.wantError {
g.Expect(len(errs)).NotTo(Equal(0))
} else {
g.Expect(len(errs)).To(Equal(0))
}
})
}
}
5 changes: 5 additions & 0 deletions pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion pkg/manager/member/startscript/v1/render_script.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@ func RenderTiDBStartScript(tc *v1alpha1.TidbCluster) (string, error) {
PluginDirectory: "/plugins",
PluginList: strings.Join(plugins, ","),
}

model.Path = "${CLUSTER_NAME}-pd:2379"
if tc.AcrossK8s() {
model.Path = "${CLUSTER_NAME}-pd:2379" // get pd addr from discovery in startup script
Expand Down
22 changes: 22 additions & 0 deletions pkg/manager/member/tidb_member_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ const (

// tidb DC label Name
tidbDCLabel = "zone"

bootstrapSQLFilePath = "/etc/tidb-bootstrap"
bootstrapSQLFileName = "bootstrap.sql"
)

var (
Expand Down Expand Up @@ -579,6 +582,9 @@ func getTiDBConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) {
config.Set("security.ssl-cert", path.Join(serverCertPath, corev1.TLSCertKey))
config.Set("security.ssl-key", path.Join(serverCertPath, corev1.TLSPrivateKeyKey))
}
if tc.Spec.TiDB.IsBootstrapSQLEnabled() {
config.Set("initialize-sql-file", path.Join(bootstrapSQLFilePath, bootstrapSQLFileName))
}
confText, err := config.MarshalTOML()
if err != nil {
return nil, err
Expand Down Expand Up @@ -767,6 +773,22 @@ func getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap)
},
})
}
if tc.Spec.TiDB != nil && tc.Spec.TiDB.IsBootstrapSQLEnabled() {
volMounts = append(volMounts, corev1.VolumeMount{
Name: "tidb-bootstrap-sql", ReadOnly: true, MountPath: bootstrapSQLFilePath,
})

vols = append(vols, corev1.Volume{
Name: "tidb-bootstrap-sql", VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: *tc.Spec.TiDB.BootstrapSQLConfigMapName,
},
Items: []corev1.KeyToPath{{Key: "bootstrap-sql", Path: bootstrapSQLFileName}},
},
},
})
}
if tc.IsTLSClusterEnabled() {
vols = append(vols, corev1.Volume{
Name: "tidb-tls", VolumeSource: corev1.VolumeSource{
Expand Down