-
Notifications
You must be signed in to change notification settings - Fork 10
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
SQLite migration #150
SQLite migration #150
Changes from 41 commits
415d668
0c8e560
560695a
ad3eb0e
13dccb7
fe8db98
330b511
b38a865
8545ced
ee7a32e
058db28
7d9bbc1
615a82d
d716a1a
eb00877
84b0eaa
54e3eff
60c9b47
e36dbf0
d004214
150226e
1a66b31
bfcc349
de57f19
1616acc
39d667e
3ab73bc
ad7f4a8
989c898
11fff11
c8a08f7
251be4d
66ee09d
a1ba600
3c20da5
e2cca3e
3e9a90e
52204ea
36c0073
2ee5c71
010d3f8
c98074b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -14,12 +14,16 @@ linters: | |
- whitespace | ||
- godot | ||
- lll | ||
- sqlclosecheck | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added this extra linter that sounds useful. |
||
|
||
issues: | ||
exclude-use-default: false | ||
|
||
exclude: | ||
- stutters | ||
|
||
run: | ||
timeout: 30m | ||
|
||
skip-dirs: | ||
- "pkg/sqlstore/impl/system/internal/db" | ||
Comment on lines
+28
to
+29
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -9,7 +9,6 @@ HTTP_PORT ?= 8080 | |
GCP_PROJECT=textile-310716 | ||
|
||
GO_BINDATA=go run github.com/go-bindata/go-bindata/v3/[email protected] | ||
SQLC=go run github.com/kyleconroy/sqlc/cmd/[email protected] | ||
GOVVV=go run github.com/ahmetb/[email protected] | ||
|
||
GOVVV_FLAGS=$(shell $(GOVVV) -flags -version $(BIN_VERSION) -pkg $(shell go list ./buildinfo)) | ||
|
@@ -32,11 +31,11 @@ ethereum-testerc721a: | |
go run github.com/ethereum/go-ethereum/cmd/[email protected] --abi ./pkg/tables/impl/ethereum/test/erc721aQueryable/abi.json --pkg erc721aQueryable --type Contract --out pkg/tables/impl/ethereum/test/erc721aQueryable/erc721aQueryable.go --bin pkg/tables/impl/ethereum/test/erc721aQueryable/bytecode.bin | ||
.PHONY: ethereum-testerc721a | ||
|
||
|
||
system-sql-assets: | ||
cd pkg/sqlstore/impl/system && $(GO_BINDATA) -pkg migrations -prefix migrations/ -o migrations/migrations.go -ignore=migrations.go migrations && $(SQLC) generate; cd -; | ||
cd pkg/sqlstore/impl/system && $(GO_BINDATA) -pkg migrations -prefix migrations/ -o migrations/migrations.go -ignore=migrations.go migrations | ||
jsign marked this conversation as resolved.
Show resolved
Hide resolved
|
||
.PHONY: system-sql-assets | ||
|
||
|
||
# Building and publishing image to GCP | ||
|
||
build-api: | ||
|
@@ -62,8 +61,8 @@ publish: | |
|
||
# Test | ||
|
||
test: | ||
go test -v ./... | ||
test: | ||
go test ./... -race | ||
Comment on lines
+64
to
+65
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We didn't have the |
||
.PHONY: test | ||
|
||
# Lint | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,21 +2,22 @@ package main | |
|
||
import ( | ||
"encoding/json" | ||
"fmt" | ||
"flag" | ||
"os" | ||
"path" | ||
"strings" | ||
|
||
"github.com/omeid/uconfig" | ||
"github.com/omeid/uconfig/plugins" | ||
"github.com/omeid/uconfig/plugins/file" | ||
"github.com/rs/zerolog/log" | ||
"github.com/textileio/go-tableland/internal/tableland" | ||
) | ||
|
||
// configFilename is the filename of the config file automatically loaded. | ||
var configFilename = "config.json" | ||
|
||
type config struct { | ||
Impl string `default:"mesa"` // service implementation (mock or mesa) | ||
HTTP struct { | ||
Port string `default:"8080"` // HTTP port (e.g. 8080) | ||
|
||
|
@@ -26,13 +27,6 @@ type config struct { | |
Gateway struct { | ||
ExternalURIPrefix string `default:"https://testnet.tableland.network"` | ||
} | ||
DB struct { | ||
Host string `default:"database"` | ||
Port string `default:"5432"` | ||
User string `default:"dev_user"` | ||
Pass string `default:"dev_password"` | ||
Name string `default:"dev_database"` | ||
} | ||
TableConstraints TableConstraints | ||
QueryConstraints QueryConstraints | ||
Metrics struct { | ||
|
@@ -42,10 +36,6 @@ type config struct { | |
Human bool `default:"false"` | ||
Debug bool `default:"false"` | ||
} | ||
AdminAPI struct { | ||
Username string `default:""` | ||
Password string `default:""` | ||
} | ||
Chains []ChainConfig | ||
} | ||
|
||
|
@@ -74,10 +64,9 @@ type ChainConfig struct { | |
PrivateKey string `default:""` | ||
} | ||
EventFeed struct { | ||
ChainAPIBackoff string `default:"15s"` | ||
MaxBlocksFetchSize int `default:"10000"` | ||
MinBlockDepth int `default:"5"` | ||
NewBlockTimeout string `default:"30s"` | ||
ChainAPIBackoff string `default:"15s"` | ||
MinBlockDepth int `default:"5"` | ||
NewBlockTimeout string `default:"30s"` | ||
} | ||
EventProcessor struct { | ||
BlockFailedExecutionBackoff string `default:"10s"` | ||
|
@@ -90,22 +79,35 @@ type ChainConfig struct { | |
} | ||
} | ||
|
||
func setupConfig() *config { | ||
fileBytes, err := os.ReadFile(configFilename) | ||
fileStr := string(fileBytes) | ||
func setupConfig() (*config, string) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Some changes here to support the new |
||
flagDirPath := flag.String("dir", "${HOME}/.tableland", "Directory where the configuration and DB exist") | ||
flag.Parse() | ||
if flagDirPath == nil { | ||
log.Fatal().Msg("--dir is null") | ||
return nil, "" // Helping the linter know the next line is safe. | ||
} | ||
dirPath := os.ExpandEnv(*flagDirPath) | ||
|
||
_ = os.MkdirAll(dirPath, 0755) | ||
Comment on lines
+83
to
+91
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Read the |
||
|
||
var plugins []plugins.Plugin | ||
if err != os.ErrNotExist { | ||
fileStr = os.ExpandEnv(fileStr) | ||
fullPath := path.Join(dirPath, configFilename) | ||
configFileBytes, err := os.ReadFile(fullPath) | ||
if os.IsNotExist(err) { | ||
log.Info().Str("configFilePath", fullPath).Msg("config file not found") | ||
} else if err != nil { | ||
log.Fatal().Str("configFilePath", fullPath).Err(err).Msg("opening config file") | ||
} else { | ||
fileStr := os.ExpandEnv(string(configFileBytes)) | ||
plugins = append(plugins, file.NewReader(strings.NewReader(fileStr), json.Unmarshal)) | ||
} | ||
|
||
conf := &config{} | ||
c, err := uconfig.Classic(&conf, file.Files{}, plugins...) | ||
if err != nil { | ||
fmt.Printf("invalid configuration: %s", err) | ||
c.Usage() | ||
os.Exit(1) | ||
log.Fatal().Err(err).Msg("invalid configuration") | ||
} | ||
|
||
return conf | ||
return conf, dirPath | ||
Comment on lines
+94
to
+112
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There was some incorrect error handling here that was fixed. |
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,6 +4,7 @@ import ( | |
"context" | ||
"fmt" | ||
"net/http" | ||
"path" | ||
"sync" | ||
"time" | ||
|
||
|
@@ -36,7 +37,7 @@ import ( | |
) | ||
|
||
func main() { | ||
config := setupConfig() | ||
config, dirPath := setupConfig() | ||
logging.SetupLogger(buildinfo.GitCommit, config.Log.Debug, config.Log.Human) | ||
|
||
// Validator instrumentation configuration. | ||
|
@@ -48,12 +49,8 @@ func main() { | |
} | ||
|
||
databaseURL := fmt.Sprintf( | ||
"postgres://%s:%s@%s:%s/%s?sslmode=disable&timezone=UTC", | ||
config.DB.User, | ||
config.DB.Pass, | ||
config.DB.Host, | ||
config.DB.Port, | ||
config.DB.Name, | ||
"file://%s?_busy_timeout=5000&_foreign_keys=on&_journal_mode=WAL", | ||
path.Join(dirPath, "database.db"), | ||
Comment on lines
+52
to
+53
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is explained in the PR description. |
||
) | ||
|
||
parserOpts := []parsing.Option{ | ||
|
@@ -112,12 +109,6 @@ func main() { | |
Handler: router.Handler(), | ||
} | ||
|
||
// Admin endpoint configuration. | ||
if config.AdminAPI.Password == "" { | ||
log.Warn(). | ||
Msg("no admin api password set") | ||
} | ||
|
||
go func() { | ||
if err := server.ListenAndServe(); err != nil { | ||
if err == http.ErrServerClosed { | ||
|
@@ -172,7 +163,7 @@ func createChainIDStack( | |
|
||
systemStore, err := sqlstoreimpl.NewInstrumentedSystemStore(config.ChainID, store) | ||
if err != nil { | ||
return chains.ChainStack{}, fmt.Errorf("instrumenting system store pgx: %s", err) | ||
return chains.ChainStack{}, fmt.Errorf("instrumenting system store: %s", err) | ||
} | ||
|
||
conn, err := ethclient.Dial(config.Registry.EthEndpoint) | ||
|
@@ -242,7 +233,6 @@ func createChainIDStack( | |
} | ||
efOpts := []eventfeed.Option{ | ||
eventfeed.WithChainAPIBackoff(chainAPIBackoff), | ||
eventfeed.WithMaxBlocksFetchSize(config.EventFeed.MaxBlocksFetchSize), | ||
eventfeed.WithMinBlockDepth(config.EventFeed.MinBlockDepth), | ||
eventfeed.WithNewBlockTimeout(newBlockTimeout), | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Cool tunning to repeat
make test
up to 3 times (if needed) if some test is flaky. On a second (or third) run the test cache will be leveraged, so only the potential flaky tests would be run again.