feat: change primary keys in tables for identity-associated data

Makefile

@@ -164,7 +164,7 @@ authors:  # updates the AUTHORS file
 # Formats the code
 .PHONY: format
 format: .bin/goimports .bin/ory node_modules
-	.bin/ory dev headers copyright --exclude=internal/httpclient --exclude=internal/client-go --exclude test/e2e/proxy/node_modules --exclude test/e2e/node_modules --exclude node_modules
+	.bin/ory dev headers copyright --exclude=internal/httpclient --exclude=internal/client-go --exclude test/e2e/proxy/node_modules --exclude test/e2e/node_modules --exclude node_modules --exclude x/slices
 	goimports -w -local github.com/ory .
 	npm exec -- prettier --write 'test/e2e/**/*{.ts,.js}'
 	npm exec -- prettier --write '.github'

driver/registry.go

@@ -7,45 +7,38 @@ import (
 	"context"
 	"io/fs"
 
-	"github.com/ory/kratos/selfservice/sessiontokenexchange"
-	"github.com/ory/x/contextx"
-	"github.com/ory/x/jsonnetsecure"
-	"github.com/ory/x/otelx"
-	prometheus "github.com/ory/x/prometheusx"
-
 	"github.com/gorilla/sessions"
 	"github.com/pkg/errors"
 
-	"github.com/ory/nosurf"
-
-	"github.com/ory/x/logrusx"
-
 	"github.com/ory/kratos/continuity"
 	"github.com/ory/kratos/courier"
+	"github.com/ory/kratos/driver/config"
 	"github.com/ory/kratos/hash"
+	"github.com/ory/kratos/identity"
+	"github.com/ory/kratos/persistence"
 	"github.com/ory/kratos/schema"
+	"github.com/ory/kratos/selfservice/errorx"
+	"github.com/ory/kratos/selfservice/flow/login"
+	"github.com/ory/kratos/selfservice/flow/logout"
 	"github.com/ory/kratos/selfservice/flow/recovery"
+	"github.com/ory/kratos/selfservice/flow/registration"
 	"github.com/ory/kratos/selfservice/flow/settings"
 	"github.com/ory/kratos/selfservice/flow/verification"
+	"github.com/ory/kratos/selfservice/sessiontokenexchange"
 	"github.com/ory/kratos/selfservice/strategy/code"
 	"github.com/ory/kratos/selfservice/strategy/link"
-
-	"github.com/ory/x/healthx"
-
-	"github.com/ory/kratos/persistence"
-	"github.com/ory/kratos/selfservice/flow/login"
-	"github.com/ory/kratos/selfservice/flow/logout"
-	"github.com/ory/kratos/selfservice/flow/registration"
-
-	"github.com/ory/kratos/x"
-
-	"github.com/ory/x/dbal"
-
-	"github.com/ory/kratos/driver/config"
-	"github.com/ory/kratos/identity"
-	"github.com/ory/kratos/selfservice/errorx"
 	password2 "github.com/ory/kratos/selfservice/strategy/password"
 	"github.com/ory/kratos/session"
+	"github.com/ory/kratos/x"
+	"github.com/ory/nosurf"
+	"github.com/ory/x/contextx"
+	"github.com/ory/x/dbal"
+	"github.com/ory/x/healthx"
+	"github.com/ory/x/jsonnetsecure"
+	"github.com/ory/x/logrusx"
+	"github.com/ory/x/otelx"
+	"github.com/ory/x/popx"
+	prometheus "github.com/ory/x/prometheusx"
 )
 
 type Registry interface {
@@ -185,6 +178,7 @@ type options struct {
 	replaceTracer           func(*otelx.Tracer) *otelx.Tracer
 	inspect                 func(Registry) error
 	extraMigrations         []fs.FS
+	extraGoMigrations       popx.Migrations
 	replacementStrategies   []NewStrategy
 	extraHooks              map[string]func(config.SelfServiceHook) any
 	disableMigrationLogging bool
@@ -244,6 +238,12 @@ func WithExtraMigrations(m ...fs.FS) RegistryOption {
 	}
 }
 
+func WithExtraGoMigrations(m ...popx.Migration) RegistryOption {
+	return func(o *options) {
+		o.extraGoMigrations = append(o.extraGoMigrations, m...)
+	}
+}
+
 func WithDisabledMigrationLogging() RegistryOption {
 	return func(o *options) {
 		o.disableMigrationLogging = true

driver/registry_default.go

@@ -656,7 +656,10 @@ func (m *RegistryDefault) Init(ctx context.Context, ctxer contextx.Contextualize
 			m.Logger().WithError(err).Warnf("Unable to open database, retrying.")
 			return errors.WithStack(err)
 		}
-		p, err := sql.NewPersister(ctx, m, c, sql.WithExtraMigrations(o.extraMigrations...), sql.WithDisabledLogging(o.disableMigrationLogging))
+		p, err := sql.NewPersister(ctx, m, c,
+			sql.WithExtraMigrations(o.extraMigrations...),
+			sql.WithExtraGoMigrations(o.extraGoMigrations...),
+			sql.WithDisabledLogging(o.disableMigrationLogging))
 		if err != nil {
 			m.Logger().WithError(err).Warnf("Unable to initialize persister, retrying.")
 			return err

go.mod

@@ -56,7 +56,7 @@ require (
 	github.com/jteeuwen/go-bindata v3.0.7+incompatible
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/knadh/koanf/parsers/json v0.1.0
-	github.com/laher/mergefs v0.1.2-0.20230223191438-d16611b2f4e7
+	github.com/laher/mergefs v0.1.2-0.20230223191438-d16611b2f4e7 // indirect
 	github.com/lestrrat-go/jwx v1.2.28 // indirect
 	github.com/luna-duclos/instrumentedsql v1.1.3
 	github.com/mailhog/MailHog v1.0.1
@@ -97,10 +97,10 @@ require (
 	go.opentelemetry.io/otel/sdk v1.21.0
 	go.opentelemetry.io/otel/trace v1.22.0
 	golang.org/x/crypto v0.18.0
-	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
+	golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3
 	golang.org/x/net v0.20.0
 	golang.org/x/oauth2 v0.16.0
-	golang.org/x/sync v0.5.0
+	golang.org/x/sync v0.6.0
 	golang.org/x/text v0.14.0
 	golang.org/x/tools/cmd/cover v0.1.0-deprecated
 	google.golang.org/grpc v1.59.0
@@ -307,7 +307,7 @@ require (
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/term v0.16.0 // indirect
-	golang.org/x/tools v0.15.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect

go.sum

@@ -712,7 +712,6 @@ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsI
 github.com/markbates/pkger v0.17.1 h1:/MKEtWqtc0mZvu9OinB9UzVN9iYCwLWuyUv4Bw+PCno=
 github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
-github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE=
 github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
@@ -1119,8 +1118,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
-golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
+golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 h1:/RIbNt/Zr7rVhIkQhooTxCxFcdWLGIKnZA4IXNFSrvo=
+golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1234,8 +1233,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1422,8 +1421,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8=
-golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
+golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/tools/cmd/cover v0.1.0-deprecated h1:Rwy+mWYz6loAF+LnG1jHG/JWMHRMMC2/1XX3Ejkx9lA=
 golang.org/x/tools/cmd/cover v0.1.0-deprecated/go.mod h1:hMDiIvlpN1NoVgmjLjUJE9tMHyxHjFX7RuQ+rW12mSA=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

identity/credentials.go

@@ -7,11 +7,16 @@ import (
 	"context"
 	"database/sql"
 	"reflect"
+	"sync"
 	"time"
 
+	"github.com/gobuffalo/pop/v6"
 	"github.com/gofrs/uuid"
+	"github.com/pkg/errors"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/ory/kratos/ui/node"
+	"github.com/ory/x/otelx"
 	"github.com/ory/x/sqlxx"
 )
 
@@ -184,11 +189,54 @@ func (c Credentials) GetID() uuid.UUID {
 	return c.ID
 }
 
+func (c Credentials) GetNID() uuid.UUID {
+	return c.NID
+}
+
+var (
+	typeTable map[uuid.UUID]CredentialsType
+	typeErr   error
+	typeOnce  sync.Once
+)
+
+func (c *Credentials) AfterFind(con *pop.Connection) error {
+	typeOnce.Do(func() {
+		span := trace.SpanFromContext(con.Context())
+		ctx, span := span.TracerProvider().Tracer("").Start(con.Context(), "identity.Credentials.AfterFind")
+		con = con.WithContext(ctx)
+		defer otelx.End(span, &typeErr)
+
+		var table []CredentialsTypeTable
+		if typeErr = con.All(&table); typeErr != nil {
+			return
+		}
+		typeTable = make(map[uuid.UUID]CredentialsType, len(table))
+		for _, t := range table {
+			typeTable[t.ID] = t.Name
+		}
+	})
+	if typeErr != nil {
+		return typeErr
+	}
+
+	var ok bool
+	c.Type, ok = typeTable[c.IdentityCredentialTypeID]
+	if !ok {
+		return errors.New("could not find credentials type")
+	}
+
+	return nil
+}
+
+var _ pop.AfterFindable = (*Credentials)(nil)
+
 type (
 	// swagger:ignore
 	CredentialIdentifier struct {
 		ID         uuid.UUID `db:"id"`
 		Identifier string    `db:"identifier"`
+		// Identity is a helper struct field for gobuffalo.pop.
+		IdentityID uuid.UUID `json:"-" db:"identity_id"`
 		// IdentityCredentialsID is a helper struct field for gobuffalo.pop.
 		IdentityCredentialsID uuid.UUID `json:"-" db:"identity_credential_id"`
 		// IdentityCredentialsTypeID is a helper struct field for gobuffalo.pop.

identity/test/pool.go

@@ -1281,8 +1281,8 @@ func TestPool(ctx context.Context, conf *config.Config, p persistence.Persister,
 			require.NoError(t, p.GetConnection(ctx).RawQuery("INSERT INTO identity_credentials (id, identity_id, nid, identity_credential_type_id, created_at, updated_at, config) VALUES (?, ?, ?, ?, ?, ?, '{}')", cid2, iid, nid2, m[0].ID, time.Now(), time.Now()).Exec())
 
 			ici1, ici2 := x.NewUUID(), x.NewUUID()
-			require.NoError(t, p.GetConnection(ctx).RawQuery("INSERT INTO identity_credential_identifiers (id, identity_credential_id, nid, identifier, created_at, updated_at, identity_credential_type_id) VALUES (?, ?, ?, ?, ?, ?, ?)", ici1, cid1, nid1, "nid1", time.Now(), time.Now(), m[0].ID).Exec())
-			require.NoError(t, p.GetConnection(ctx).RawQuery("INSERT INTO identity_credential_identifiers (id, identity_credential_id, nid, identifier, created_at, updated_at, identity_credential_type_id) VALUES (?, ?, ?, ?, ?, ?, ?)", ici2, cid2, nid2, "nid2", time.Now(), time.Now(), m[0].ID).Exec())
+			require.NoError(t, p.GetConnection(ctx).RawQuery("INSERT INTO identity_credential_identifiers (id, identity_id, identity_credential_id, nid, identifier, created_at, updated_at, identity_credential_type_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ici1, iid, cid1, nid1, "nid1", time.Now(), time.Now(), m[0].ID).Exec())
+			require.NoError(t, p.GetConnection(ctx).RawQuery("INSERT INTO identity_credential_identifiers (id, identity_id, identity_credential_id, nid, identifier, created_at, updated_at, identity_credential_type_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ici2, iid, cid2, nid2, "nid2", time.Now(), time.Now(), m[0].ID).Exec())
 
 			_, err := p.GetIdentity(ctx, nid1, identity.ExpandNothing)
 			require.ErrorIs(t, err, sqlcon.ErrNoRows)

persistence/sql/identity/persister_identity.go

@@ -331,6 +331,7 @@ func (p *IdentityPersister) createIdentityCredentials(ctx context.Context, conn
 
 			identifiers = append(identifiers, &identity.CredentialIdentifier{
 				Identifier:                ids,
+				IdentityID:                cred.IdentityID,
 				IdentityCredentialsID:     cred.ID,
 				IdentityCredentialsTypeID: ct.ID,
 				NID:                       p.NetworkID(ctx),
@@ -656,7 +657,7 @@ func QueryForCredentials(con *pop.Connection, where ...Where) (map[uuid.UUID](ma
 	ici := "identity_credential_identifiers"
 	switch con.Dialect.Name() {
 	case "cockroach":
-		ici += "@identity_credential_identifiers_nid_identity_credential_id_idx"
+		ici += "@primary"
 	case "sqlite3":
 		ici += " INDEXED BY identity_credential_identifiers_nid_identity_credential_id_idx"
 	case "mysql":
@@ -680,7 +681,9 @@ func QueryForCredentials(con *pop.Connection, where ...Where) (map[uuid.UUID](ma
 		"(identity_credentials.identity_credential_type_id = ict.id)",
 	).LeftJoin(
 		ici,
-		"identity_credential_identifiers.identity_credential_id = identity_credentials.id AND identity_credential_identifiers.nid = identity_credentials.nid",
+		`identity_credential_identifiers.identity_id = identity_credentials.identity_id
+		AND identity_credential_identifiers.identity_credential_id = identity_credentials.id
+		AND identity_credential_identifiers.nid = identity_credentials.nid`,
 	)
 	for _, w := range where {
 		q = q.Where("("+w.Condition+")", w.Args...)

persistence/sql/migratest/migration_test.go

@@ -16,12 +16,14 @@ import (
 	"github.com/ory/x/servicelocatorx"
 
 	"github.com/ory/kratos/identity"
+	"github.com/ory/kratos/persistence/sql/migrations/gomigrations"
 
 	"github.com/bradleyjkemp/cupaloy/v2"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/ory/x/dbal"
 
+	"github.com/ory/kratos/x/slices"
 	"github.com/ory/kratos/x/xsql"
 
 	"github.com/ory/x/migratest"
@@ -108,7 +110,6 @@ func TestMigrations_Cockroach(t *testing.T) {
 
 func testDatabase(t *testing.T, db string, c *pop.Connection) {
 	ctx := context.Background()
-	l := logrusx.New("", "", logrusx.ForceLevel(logrus.ErrorLevel))
 
 	t.Logf("Cleaning up before migrations")
 	_ = os.Remove("../migrations/sql/schema.sql")
@@ -130,15 +131,18 @@ func testDatabase(t *testing.T, db string, c *pop.Connection) {
 	}
 	t.Logf("URL: %s", url)
 
-	t.Run("suite=up", func(t *testing.T) {
-		tm, err := popx.NewMigrationBox(
-			os.DirFS("../migrations/sql"),
-			popx.NewMigrator(c, logrusx.New("", "", logrusx.ForceLevel(logrus.DebugLevel)), nil, 1*time.Minute),
-			popx.WithTestdata(t, os.DirFS("./testdata")),
-		)
-		require.NoError(t, err)
-		require.NoError(t, tm.Up(ctx))
-	})
+	tm, err := popx.NewMigrationBox(
+		os.DirFS("../migrations/sql"),
+		popx.NewMigrator(c, logrusx.New("", "", logrusx.ForceLevel(logrus.DebugLevel)), nil, 1*time.Minute),
+		popx.WithTestdata(t, os.DirFS("./testdata")),
+		popx.WithGoMigrations(slices.Concat(
+			gomigrations.IdentityPrimaryKeysStep1,
+			gomigrations.IdentityPrimaryKeysStep2,
+		)),
+	)
+	require.NoError(t, err)
+	tm.DumpMigrations = true
+	require.NoError(t, tm.Up(ctx))
 
 	t.Run("suite=fixtures", func(t *testing.T) {
 		wg := &sync.WaitGroup{}
@@ -423,8 +427,6 @@ func testDatabase(t *testing.T, db string, c *pop.Connection) {
 		})
 	})
 
-	t.Run("suite=down", func(t *testing.T) {
-		tm := popx.NewTestMigrator(t, c, os.DirFS("../migrations/sql"), os.DirFS("./testdata"), l)
-		require.NoError(t, tm.Down(ctx, -1))
-	})
+	tm.DumpMigrations = false
+	require.NoError(t, tm.Down(ctx, -1))
 }