diff --git a/.dockerignore b/.dockerignore index 7a790643..b1a15ac9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,10 @@ - -# Created by https://www.toptal.com/developers/gitignore/api/go,visualstudiocode,intellij,macos,windows,linux -# Edit at https://www.toptal.com/developers/gitignore?templates=go,visualstudiocode,intellij,macos,windows,linux +# Created by https://www.toptal.com/developers/gitignore/api/go,goland,visualstudiocode,vim,macos,windows,linux +# Edit at https://www.toptal.com/developers/gitignore?templates=go,goland,visualstudiocode,vim,macos,windows,linux ### Go ### +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# # Binaries for programs and plugins *.exe *.exe~ @@ -19,11 +21,10 @@ # Dependency directories (remove the comment below to include it) # vendor/ -### Go Patch ### -/vendor/ -/Godeps/ +# Go workspace file +go.work -### Intellij ### +### GoLand ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 @@ -87,6 +88,9 @@ atlassian-ide-plugin.xml # Cursive Clojure plugin .idea/replstate.xml +# SonarLint plugin +.idea/sonarlint/ + # Crashlytics plugin (for Android Studio and IntelliJ) com_crashlytics_export_strings.xml crashlytics.properties @@ -99,7 +103,7 @@ fabric.properties # Android studio 3.1+ serialized cache file .idea/caches/build_file_checksums.ser -### Intellij Patch ### +### GoLand Patch ### # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 # *.iml @@ -129,6 +133,10 @@ fabric.properties # https://plugins.jetbrains.com/plugin/12206-codestream .idea/codestream.xml +# Azure Toolkit for IntelliJ plugin +# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij +.idea/**/azureSettings.xml + ### Linux ### *~ @@ -173,17 +181,44 @@ Network Trash Folder Temporary Items .apdisk +### macOS Patch ### +# iCloud generated files +*.icloud + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + ### VisualStudioCode ### .vscode/* !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json -*.code-workspace +!.vscode/*.code-snippets # Local History for Visual Studio Code .history/ +# Built Visual Studio Code Extensions +*.vsix + ### VisualStudioCode Patch ### # Ignore all local history of files .history @@ -215,15 +250,7 @@ $RECYCLE.BIN/ # Windows shortcuts *.lnk -# End of https://www.toptal.com/developers/gitignore/api/go,visualstudiocode,intellij,macos,windows,linux +# End of https://www.toptal.com/developers/gitignore/api/go,goland,visualstudiocode,vim,macos,windows,linux -errors* -dist/ nebula - -node_modules/ - -plot.png -sessions.csv -analysis .git \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3d04a004..5a2a2e8b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ - -# Created by https://www.toptal.com/developers/gitignore/api/go,visualstudiocode,intellij,macos,windows,linux -# Edit at https://www.toptal.com/developers/gitignore?templates=go,visualstudiocode,intellij,macos,windows,linux +# Created by https://www.toptal.com/developers/gitignore/api/go,goland,visualstudiocode,vim,macos,windows,linux +# Edit at https://www.toptal.com/developers/gitignore?templates=go,goland,visualstudiocode,vim,macos,windows,linux ### Go ### +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# # Binaries for programs and plugins *.exe *.exe~ @@ -19,16 +21,121 @@ # Dependency directories (remove the comment below to include it) # vendor/ -### Go Patch ### -/vendor/ -/Godeps/ +# Go workspace file +go.work -### Intellij ### +### GoLand ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff -.idea/ +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### GoLand Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +# Azure Toolkit for IntelliJ plugin +# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij +.idea/**/azureSettings.xml ### Linux ### *~ @@ -74,17 +181,44 @@ Network Trash Folder Temporary Items .apdisk +### macOS Patch ### +# iCloud generated files +*.icloud + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + ### VisualStudioCode ### .vscode/* !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json -*.code-workspace +!.vscode/*.code-snippets # Local History for Visual Studio Code .history/ +# Built Visual Studio Code Extensions +*.vsix + ### VisualStudioCode Patch ### # Ignore all local history of files .history @@ -116,70 +250,8 @@ $RECYCLE.BIN/ # Windows shortcuts *.lnk -# End of https://www.toptal.com/developers/gitignore/api/go,visualstudiocode,intellij,macos,windows,linux - -dist/ -./nebula - -node_modules/ - -plot.png -sessions.csv -sessions_filecoin.csv - -*__pycache__* - -analysis/provide/data -analysis/provide/server - -**/.cache/ -**/plots-*/ - -**/*.vault -**/ansible*cfg -udgerdb_v3.dat - - -# Created by https://www.toptal.com/developers/gitignore/api/terraform -# Edit at https://www.toptal.com/developers/gitignore?templates=terraform - -### Terraform ### -# Local .terraform directories -**/.terraform/* - -# .tfstate files -*.tfstate -*.tfstate.* - -# Crash log files -crash.log -crash.*.log - -# Exclude all .tfvars files, which are likely to contain sensitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject -# to change depending on the environment. -*.tfvars -*.tfvars.json - -# Ignore override files as they are usually used to override resources locally and so -# are not checked in -override.tf -override.tf.json -*_override.tf -*_override.tf.json - -# Include override files you do wish to add to version control using negated pattern -# !example_override.tf - -# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan -# example: *tfplan* - -# Ignore CLI configuration files -.terraformrc -terraform.rc +# End of https://www.toptal.com/developers/gitignore/api/go,goland,visualstudiocode,vim,macos,windows,linux -# End of https://www.toptal.com/developers/gitignore/api/terraform out -report/db.toml .tool-versions +**/*.dat diff --git a/cmd/nebula/cmd.go b/cmd/nebula/cmd.go index 04f2500b..3d2f011e 100644 --- a/cmd/nebula/cmd.go +++ b/cmd/nebula/cmd.go @@ -13,8 +13,8 @@ import ( "github.com/urfave/cli/v2" "github.com/volatiletech/sqlboiler/v4/boil" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/metrics" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/metrics" ) const ( diff --git a/cmd/nebula/cmd_crawl.go b/cmd/nebula/cmd_crawl.go index 7cd79e86..3ab40859 100644 --- a/cmd/nebula/cmd_crawl.go +++ b/cmd/nebula/cmd_crawl.go @@ -17,12 +17,12 @@ import ( "github.com/urfave/cli/v2" "github.com/volatiletech/null/v8" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/core" - "github.com/dennis-tra/nebula-crawler/pkg/db" - "github.com/dennis-tra/nebula-crawler/pkg/discv5" - "github.com/dennis-tra/nebula-crawler/pkg/libp2p" - "github.com/dennis-tra/nebula-crawler/pkg/models" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/core" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/db/models" + "github.com/dennis-tra/nebula-crawler/discv5" + "github.com/dennis-tra/nebula-crawler/libp2p" ) var crawlConfig = &config.Crawl{ diff --git a/cmd/nebula/cmd_monitor.go b/cmd/nebula/cmd_monitor.go index 09aac799..714473c3 100644 --- a/cmd/nebula/cmd_monitor.go +++ b/cmd/nebula/cmd_monitor.go @@ -10,11 +10,11 @@ import ( log "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/core" - "github.com/dennis-tra/nebula-crawler/pkg/db" - "github.com/dennis-tra/nebula-crawler/pkg/discv5" - "github.com/dennis-tra/nebula-crawler/pkg/libp2p" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/core" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/discv5" + "github.com/dennis-tra/nebula-crawler/libp2p" ) var monitorConfig = &config.Monitor{ diff --git a/cmd/nebula/cmd_networks.go b/cmd/nebula/cmd_networks.go index f1f609cb..e1a79993 100644 --- a/cmd/nebula/cmd_networks.go +++ b/cmd/nebula/cmd_networks.go @@ -4,7 +4,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" - "github.com/dennis-tra/nebula-crawler/pkg/config" + "github.com/dennis-tra/nebula-crawler/config" ) // NetworksCommand contains the networks sub-command configuration. diff --git a/cmd/nebula/cmd_resolve.go b/cmd/nebula/cmd_resolve.go index ee712ba3..fde46350 100644 --- a/cmd/nebula/cmd_resolve.go +++ b/cmd/nebula/cmd_resolve.go @@ -13,11 +13,11 @@ import ( "github.com/volatiletech/null/v8" "github.com/volatiletech/sqlboiler/v4/boil" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/db" - "github.com/dennis-tra/nebula-crawler/pkg/maxmind" - "github.com/dennis-tra/nebula-crawler/pkg/models" - "github.com/dennis-tra/nebula-crawler/pkg/udger" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/db/models" + "github.com/dennis-tra/nebula-crawler/maxmind" + "github.com/dennis-tra/nebula-crawler/udger" ) var resolveConfig = &config.Resolve{ diff --git a/pkg/config/bootstrap.go b/config/bootstrap.go similarity index 100% rename from pkg/config/bootstrap.go rename to config/bootstrap.go diff --git a/pkg/config/config.go b/config/config.go similarity index 100% rename from pkg/config/config.go rename to config/config.go diff --git a/pkg/core/core.go b/core/core.go similarity index 98% rename from pkg/core/core.go rename to core/core.go index 16624b41..1704d675 100644 --- a/pkg/core/core.go +++ b/core/core.go @@ -8,7 +8,7 @@ import ( "github.com/multiformats/go-multiaddr" log "github.com/sirupsen/logrus" - "github.com/dennis-tra/nebula-crawler/pkg/db" + "github.com/dennis-tra/nebula-crawler/db" ) // PeerInfo is the interface that any peer information struct must conform to. diff --git a/pkg/core/core_test.go b/core/core_test.go similarity index 98% rename from pkg/core/core_test.go rename to core/core_test.go index c330f0a3..d4675e5b 100644 --- a/pkg/core/core_test.go +++ b/core/core_test.go @@ -9,7 +9,7 @@ import ( ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/mock" - "github.com/dennis-tra/nebula-crawler/pkg/utils" + "github.com/dennis-tra/nebula-crawler/utils" ) // interface assertions diff --git a/pkg/core/engine.go b/core/engine.go similarity index 99% rename from pkg/core/engine.go rename to core/engine.go index 43c37c06..474c40f8 100644 --- a/pkg/core/engine.go +++ b/core/engine.go @@ -7,9 +7,9 @@ import ( ma "github.com/multiformats/go-multiaddr" log "github.com/sirupsen/logrus" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/metrics" - "github.com/dennis-tra/nebula-crawler/pkg/utils" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/metrics" + "github.com/dennis-tra/nebula-crawler/utils" ) // The EngineConfig object configures the core Nebula [Engine] below. diff --git a/pkg/core/engine_test.go b/core/engine_test.go similarity index 98% rename from pkg/core/engine_test.go rename to core/engine_test.go index 7cdc027f..7e80e330 100644 --- a/pkg/core/engine_test.go +++ b/core/engine_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/libp2p/go-libp2p/core/peer" ma "github.com/multiformats/go-multiaddr" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -11,10 +12,9 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/goleak" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/db" - "github.com/dennis-tra/nebula-crawler/pkg/nebtest" - "github.com/libp2p/go-libp2p/core/peer" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/nebtest" ) func TestEngineConfig_Validate(t *testing.T) { diff --git a/pkg/core/handler_crawl.go b/core/handler_crawl.go similarity index 99% rename from pkg/core/handler_crawl.go rename to core/handler_crawl.go index b0e5bf78..57d6c6e3 100644 --- a/pkg/core/handler_crawl.go +++ b/core/handler_crawl.go @@ -8,7 +8,7 @@ import ( "github.com/libp2p/go-libp2p/core/peer" log "github.com/sirupsen/logrus" - "github.com/dennis-tra/nebula-crawler/pkg/models" + "github.com/dennis-tra/nebula-crawler/db/models" ) // CrawlResult captures data that is gathered from crawling a single peer. diff --git a/pkg/core/handler_dial.go b/core/handler_dial.go similarity index 100% rename from pkg/core/handler_dial.go rename to core/handler_dial.go diff --git a/pkg/core/queue.go b/core/queue.go similarity index 100% rename from pkg/core/queue.go rename to core/queue.go diff --git a/pkg/core/queue_test.go b/core/queue_test.go similarity index 100% rename from pkg/core/queue_test.go rename to core/queue_test.go diff --git a/pkg/core/worker.go b/core/worker.go similarity index 100% rename from pkg/core/worker.go rename to core/worker.go diff --git a/pkg/core/worker_test.go b/core/worker_test.go similarity index 100% rename from pkg/core/worker_test.go rename to core/worker_test.go diff --git a/pkg/core/writer_crawl.go b/core/writer_crawl.go similarity index 93% rename from pkg/core/writer_crawl.go rename to core/writer_crawl.go index c24fd894..cf110323 100644 --- a/pkg/core/writer_crawl.go +++ b/core/writer_crawl.go @@ -8,9 +8,9 @@ import ( log "github.com/sirupsen/logrus" "github.com/volatiletech/null/v8" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/db" - "github.com/dennis-tra/nebula-crawler/pkg/utils" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/utils" ) type CrawlWriterConfig struct { diff --git a/pkg/core/writer_dial.go b/core/writer_dial.go similarity index 95% rename from pkg/core/writer_dial.go rename to core/writer_dial.go index 14e5e508..1901c33d 100644 --- a/pkg/core/writer_dial.go +++ b/core/writer_dial.go @@ -6,8 +6,8 @@ import ( log "github.com/sirupsen/logrus" - "github.com/dennis-tra/nebula-crawler/pkg/db" - "github.com/dennis-tra/nebula-crawler/pkg/models" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/db/models" ) // DialResult captures data that is gathered from pinging a single peer. diff --git a/pkg/db/client.go b/db/client.go similarity index 95% rename from pkg/db/client.go rename to db/client.go index 96b8ec88..2f7c0799 100644 --- a/pkg/db/client.go +++ b/db/client.go @@ -10,8 +10,8 @@ import ( ma "github.com/multiformats/go-multiaddr" "github.com/volatiletech/null/v8" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/models" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/db/models" ) type Client interface { diff --git a/pkg/db/client_db.go b/db/client_db.go similarity index 99% rename from pkg/db/client_db.go rename to db/client_db.go index 3915642f..308c08d3 100644 --- a/pkg/db/client_db.go +++ b/db/client_db.go @@ -34,10 +34,10 @@ import ( "github.com/volatiletech/sqlboiler/v4/queries/qm" "github.com/volatiletech/sqlboiler/v4/types" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/metrics" - "github.com/dennis-tra/nebula-crawler/pkg/models" - "github.com/dennis-tra/nebula-crawler/pkg/utils" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/db/models" + "github.com/dennis-tra/nebula-crawler/metrics" + "github.com/dennis-tra/nebula-crawler/utils" ) //go:embed migrations diff --git a/pkg/db/client_db_test.go b/db/client_db_test.go similarity index 99% rename from pkg/db/client_db_test.go rename to db/client_db_test.go index 39ae6d97..ec7f767e 100644 --- a/pkg/db/client_db_test.go +++ b/db/client_db_test.go @@ -15,8 +15,8 @@ import ( "github.com/volatiletech/sqlboiler/v4/boil" "github.com/volatiletech/sqlboiler/v4/queries/qm" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/models" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/db/models" ) func clearDatabase(ctx context.Context, db *sql.DB) error { diff --git a/pkg/db/client_json.go b/db/client_json.go similarity index 98% rename from pkg/db/client_json.go rename to db/client_json.go index 7b0aec03..023e98d8 100644 --- a/pkg/db/client_json.go +++ b/db/client_json.go @@ -14,7 +14,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/volatiletech/null/v8" - "github.com/dennis-tra/nebula-crawler/pkg/models" + "github.com/dennis-tra/nebula-crawler/db/models" ) type JSONClient struct { diff --git a/pkg/db/client_noop.go b/db/client_noop.go similarity index 96% rename from pkg/db/client_noop.go rename to db/client_noop.go index 009962dd..774dfe6b 100644 --- a/pkg/db/client_noop.go +++ b/db/client_noop.go @@ -8,7 +8,7 @@ import ( ma "github.com/multiformats/go-multiaddr" "github.com/volatiletech/null/v8" - "github.com/dennis-tra/nebula-crawler/pkg/models" + "github.com/dennis-tra/nebula-crawler/db/models" ) type NoopClient struct{} diff --git a/pkg/db/errors.go b/db/errors.go similarity index 98% rename from pkg/db/errors.go rename to db/errors.go index 78487b5b..d1c72fca 100644 --- a/pkg/db/errors.go +++ b/db/errors.go @@ -6,7 +6,7 @@ import ( "github.com/libp2p/go-libp2p/p2p/net/swarm" - "github.com/dennis-tra/nebula-crawler/pkg/models" + "github.com/dennis-tra/nebula-crawler/db/models" ) // KnownErrors contains a list of known errors. Property key + string to match for diff --git a/pkg/db/errors_test.go b/db/errors_test.go similarity index 100% rename from pkg/db/errors_test.go rename to db/errors_test.go diff --git a/pkg/db/migrations/000001_create_crawls_table.down.sql b/db/migrations/000001_create_crawls_table.down.sql similarity index 100% rename from pkg/db/migrations/000001_create_crawls_table.down.sql rename to db/migrations/000001_create_crawls_table.down.sql diff --git a/pkg/db/migrations/000001_create_crawls_table.up.sql b/db/migrations/000001_create_crawls_table.up.sql similarity index 100% rename from pkg/db/migrations/000001_create_crawls_table.up.sql rename to db/migrations/000001_create_crawls_table.up.sql diff --git a/pkg/db/migrations/000002_create_agent_versions_table.down.sql b/db/migrations/000002_create_agent_versions_table.down.sql similarity index 100% rename from pkg/db/migrations/000002_create_agent_versions_table.down.sql rename to db/migrations/000002_create_agent_versions_table.down.sql diff --git a/pkg/db/migrations/000002_create_agent_versions_table.up.sql b/db/migrations/000002_create_agent_versions_table.up.sql similarity index 100% rename from pkg/db/migrations/000002_create_agent_versions_table.up.sql rename to db/migrations/000002_create_agent_versions_table.up.sql diff --git a/pkg/db/migrations/000003_create_upsert_agent_versions_function.down.sql b/db/migrations/000003_create_upsert_agent_versions_function.down.sql similarity index 100% rename from pkg/db/migrations/000003_create_upsert_agent_versions_function.down.sql rename to db/migrations/000003_create_upsert_agent_versions_function.down.sql diff --git a/pkg/db/migrations/000003_create_upsert_agent_versions_function.up.sql b/db/migrations/000003_create_upsert_agent_versions_function.up.sql similarity index 100% rename from pkg/db/migrations/000003_create_upsert_agent_versions_function.up.sql rename to db/migrations/000003_create_upsert_agent_versions_function.up.sql diff --git a/pkg/db/migrations/000004_create_protocols_table.down.sql b/db/migrations/000004_create_protocols_table.down.sql similarity index 100% rename from pkg/db/migrations/000004_create_protocols_table.down.sql rename to db/migrations/000004_create_protocols_table.down.sql diff --git a/pkg/db/migrations/000004_create_protocols_table.up.sql b/db/migrations/000004_create_protocols_table.up.sql similarity index 100% rename from pkg/db/migrations/000004_create_protocols_table.up.sql rename to db/migrations/000004_create_protocols_table.up.sql diff --git a/pkg/db/migrations/000005_create_upsert_protocols_function.down.sql b/db/migrations/000005_create_upsert_protocols_function.down.sql similarity index 100% rename from pkg/db/migrations/000005_create_upsert_protocols_function.down.sql rename to db/migrations/000005_create_upsert_protocols_function.down.sql diff --git a/pkg/db/migrations/000005_create_upsert_protocols_function.up.sql b/db/migrations/000005_create_upsert_protocols_function.up.sql similarity index 100% rename from pkg/db/migrations/000005_create_upsert_protocols_function.up.sql rename to db/migrations/000005_create_upsert_protocols_function.up.sql diff --git a/pkg/db/migrations/000006_create_protocols_set_table.down.sql b/db/migrations/000006_create_protocols_set_table.down.sql similarity index 100% rename from pkg/db/migrations/000006_create_protocols_set_table.down.sql rename to db/migrations/000006_create_protocols_set_table.down.sql diff --git a/pkg/db/migrations/000006_create_protocols_set_table.up.sql b/db/migrations/000006_create_protocols_set_table.up.sql similarity index 100% rename from pkg/db/migrations/000006_create_protocols_set_table.up.sql rename to db/migrations/000006_create_protocols_set_table.up.sql diff --git a/pkg/db/migrations/000007_create_upsert_protocols_set_id_function.down.sql b/db/migrations/000007_create_upsert_protocols_set_id_function.down.sql similarity index 100% rename from pkg/db/migrations/000007_create_upsert_protocols_set_id_function.down.sql rename to db/migrations/000007_create_upsert_protocols_set_id_function.down.sql diff --git a/pkg/db/migrations/000007_create_upsert_protocols_set_id_function.up.sql b/db/migrations/000007_create_upsert_protocols_set_id_function.up.sql similarity index 100% rename from pkg/db/migrations/000007_create_upsert_protocols_set_id_function.up.sql rename to db/migrations/000007_create_upsert_protocols_set_id_function.up.sql diff --git a/pkg/db/migrations/000008_create_multi_addresses_table.down.sql b/db/migrations/000008_create_multi_addresses_table.down.sql similarity index 100% rename from pkg/db/migrations/000008_create_multi_addresses_table.down.sql rename to db/migrations/000008_create_multi_addresses_table.down.sql diff --git a/pkg/db/migrations/000008_create_multi_addresses_table.up.sql b/db/migrations/000008_create_multi_addresses_table.up.sql similarity index 100% rename from pkg/db/migrations/000008_create_multi_addresses_table.up.sql rename to db/migrations/000008_create_multi_addresses_table.up.sql diff --git a/pkg/db/migrations/000009_create_upsert_multi_addresses_function.down.sql b/db/migrations/000009_create_upsert_multi_addresses_function.down.sql similarity index 100% rename from pkg/db/migrations/000009_create_upsert_multi_addresses_function.down.sql rename to db/migrations/000009_create_upsert_multi_addresses_function.down.sql diff --git a/pkg/db/migrations/000009_create_upsert_multi_addresses_function.up.sql b/db/migrations/000009_create_upsert_multi_addresses_function.up.sql similarity index 100% rename from pkg/db/migrations/000009_create_upsert_multi_addresses_function.up.sql rename to db/migrations/000009_create_upsert_multi_addresses_function.up.sql diff --git a/pkg/db/migrations/000010_create_ip_addresses_table.down.sql b/db/migrations/000010_create_ip_addresses_table.down.sql similarity index 100% rename from pkg/db/migrations/000010_create_ip_addresses_table.down.sql rename to db/migrations/000010_create_ip_addresses_table.down.sql diff --git a/pkg/db/migrations/000010_create_ip_addresses_table.up.sql b/db/migrations/000010_create_ip_addresses_table.up.sql similarity index 100% rename from pkg/db/migrations/000010_create_ip_addresses_table.up.sql rename to db/migrations/000010_create_ip_addresses_table.up.sql diff --git a/pkg/db/migrations/000011_create_net_error_type.down.sql b/db/migrations/000011_create_net_error_type.down.sql similarity index 100% rename from pkg/db/migrations/000011_create_net_error_type.down.sql rename to db/migrations/000011_create_net_error_type.down.sql diff --git a/pkg/db/migrations/000011_create_net_error_type.up.sql b/db/migrations/000011_create_net_error_type.up.sql similarity index 100% rename from pkg/db/migrations/000011_create_net_error_type.up.sql rename to db/migrations/000011_create_net_error_type.up.sql diff --git a/pkg/db/migrations/000012_create_crawl_properties.down.sql b/db/migrations/000012_create_crawl_properties.down.sql similarity index 100% rename from pkg/db/migrations/000012_create_crawl_properties.down.sql rename to db/migrations/000012_create_crawl_properties.down.sql diff --git a/pkg/db/migrations/000012_create_crawl_properties.up.sql b/db/migrations/000012_create_crawl_properties.up.sql similarity index 100% rename from pkg/db/migrations/000012_create_crawl_properties.up.sql rename to db/migrations/000012_create_crawl_properties.up.sql diff --git a/pkg/db/migrations/000013_create_peers_table.down.sql b/db/migrations/000013_create_peers_table.down.sql similarity index 100% rename from pkg/db/migrations/000013_create_peers_table.down.sql rename to db/migrations/000013_create_peers_table.down.sql diff --git a/pkg/db/migrations/000013_create_peers_table.up.sql b/db/migrations/000013_create_peers_table.up.sql similarity index 100% rename from pkg/db/migrations/000013_create_peers_table.up.sql rename to db/migrations/000013_create_peers_table.up.sql diff --git a/pkg/db/migrations/000014_create_peer_logs_table.down.sql b/db/migrations/000014_create_peer_logs_table.down.sql similarity index 100% rename from pkg/db/migrations/000014_create_peer_logs_table.down.sql rename to db/migrations/000014_create_peer_logs_table.down.sql diff --git a/pkg/db/migrations/000014_create_peer_logs_table.up.sql b/db/migrations/000014_create_peer_logs_table.up.sql similarity index 100% rename from pkg/db/migrations/000014_create_peer_logs_table.up.sql rename to db/migrations/000014_create_peer_logs_table.up.sql diff --git a/pkg/db/migrations/000015_create_upsert_peer_function.down.sql b/db/migrations/000015_create_upsert_peer_function.down.sql similarity index 100% rename from pkg/db/migrations/000015_create_upsert_peer_function.down.sql rename to db/migrations/000015_create_upsert_peer_function.down.sql diff --git a/pkg/db/migrations/000015_create_upsert_peer_function.up.sql b/db/migrations/000015_create_upsert_peer_function.up.sql similarity index 100% rename from pkg/db/migrations/000015_create_upsert_peer_function.up.sql rename to db/migrations/000015_create_upsert_peer_function.up.sql diff --git a/pkg/db/migrations/000016_create_neighbors_table.down.sql b/db/migrations/000016_create_neighbors_table.down.sql similarity index 100% rename from pkg/db/migrations/000016_create_neighbors_table.down.sql rename to db/migrations/000016_create_neighbors_table.down.sql diff --git a/pkg/db/migrations/000016_create_neighbors_table.up.sql b/db/migrations/000016_create_neighbors_table.up.sql similarity index 100% rename from pkg/db/migrations/000016_create_neighbors_table.up.sql rename to db/migrations/000016_create_neighbors_table.up.sql diff --git a/pkg/db/migrations/000017_create_insert_neighbors_function.down.sql b/db/migrations/000017_create_insert_neighbors_function.down.sql similarity index 100% rename from pkg/db/migrations/000017_create_insert_neighbors_function.down.sql rename to db/migrations/000017_create_insert_neighbors_function.down.sql diff --git a/pkg/db/migrations/000017_create_insert_neighbors_function.up.sql b/db/migrations/000017_create_insert_neighbors_function.up.sql similarity index 100% rename from pkg/db/migrations/000017_create_insert_neighbors_function.up.sql rename to db/migrations/000017_create_insert_neighbors_function.up.sql diff --git a/pkg/db/migrations/000018_create_peers_x_multi_addresses_table.down.sql b/db/migrations/000018_create_peers_x_multi_addresses_table.down.sql similarity index 100% rename from pkg/db/migrations/000018_create_peers_x_multi_addresses_table.down.sql rename to db/migrations/000018_create_peers_x_multi_addresses_table.down.sql diff --git a/pkg/db/migrations/000018_create_peers_x_multi_addresses_table.up.sql b/db/migrations/000018_create_peers_x_multi_addresses_table.up.sql similarity index 100% rename from pkg/db/migrations/000018_create_peers_x_multi_addresses_table.up.sql rename to db/migrations/000018_create_peers_x_multi_addresses_table.up.sql diff --git a/pkg/db/migrations/000019_create_sessions_table.down.sql b/db/migrations/000019_create_sessions_table.down.sql similarity index 100% rename from pkg/db/migrations/000019_create_sessions_table.down.sql rename to db/migrations/000019_create_sessions_table.down.sql diff --git a/pkg/db/migrations/000019_create_sessions_table.up.sql b/db/migrations/000019_create_sessions_table.up.sql similarity index 100% rename from pkg/db/migrations/000019_create_sessions_table.up.sql rename to db/migrations/000019_create_sessions_table.up.sql diff --git a/pkg/db/migrations/000020_create_upsert_session_function.down.sql b/db/migrations/000020_create_upsert_session_function.down.sql similarity index 100% rename from pkg/db/migrations/000020_create_upsert_session_function.down.sql rename to db/migrations/000020_create_upsert_session_function.down.sql diff --git a/pkg/db/migrations/000020_create_upsert_session_function.up.sql b/db/migrations/000020_create_upsert_session_function.up.sql similarity index 100% rename from pkg/db/migrations/000020_create_upsert_session_function.up.sql rename to db/migrations/000020_create_upsert_session_function.up.sql diff --git a/pkg/db/migrations/000021_create_visits_table.down.sql b/db/migrations/000021_create_visits_table.down.sql similarity index 100% rename from pkg/db/migrations/000021_create_visits_table.down.sql rename to db/migrations/000021_create_visits_table.down.sql diff --git a/pkg/db/migrations/000021_create_visits_table.up.sql b/db/migrations/000021_create_visits_table.up.sql similarity index 100% rename from pkg/db/migrations/000021_create_visits_table.up.sql rename to db/migrations/000021_create_visits_table.up.sql diff --git a/pkg/db/migrations/000022_create_insert_visits_function.down.sql b/db/migrations/000022_create_insert_visits_function.down.sql similarity index 100% rename from pkg/db/migrations/000022_create_insert_visits_function.down.sql rename to db/migrations/000022_create_insert_visits_function.down.sql diff --git a/pkg/db/migrations/000022_create_insert_visits_function.up.sql b/db/migrations/000022_create_insert_visits_function.up.sql similarity index 100% rename from pkg/db/migrations/000022_create_insert_visits_function.up.sql rename to db/migrations/000022_create_insert_visits_function.up.sql diff --git a/pkg/db/migrations/000023_add_exposed_api_data.down.sql b/db/migrations/000023_add_exposed_api_data.down.sql similarity index 100% rename from pkg/db/migrations/000023_add_exposed_api_data.down.sql rename to db/migrations/000023_add_exposed_api_data.down.sql diff --git a/pkg/db/migrations/000023_add_exposed_api_data.up.sql b/db/migrations/000023_add_exposed_api_data.up.sql similarity index 100% rename from pkg/db/migrations/000023_add_exposed_api_data.up.sql rename to db/migrations/000023_add_exposed_api_data.up.sql diff --git a/pkg/db/migrations/000024_add_remaining_column_to_crawls.down.sql b/db/migrations/000024_add_remaining_column_to_crawls.down.sql similarity index 100% rename from pkg/db/migrations/000024_add_remaining_column_to_crawls.down.sql rename to db/migrations/000024_add_remaining_column_to_crawls.down.sql diff --git a/pkg/db/migrations/000024_add_remaining_column_to_crawls.up.sql b/db/migrations/000024_add_remaining_column_to_crawls.up.sql similarity index 100% rename from pkg/db/migrations/000024_add_remaining_column_to_crawls.up.sql rename to db/migrations/000024_add_remaining_column_to_crawls.up.sql diff --git a/pkg/db/migrations/000025_add_peer_properties_column.down.sql b/db/migrations/000025_add_peer_properties_column.down.sql similarity index 100% rename from pkg/db/migrations/000025_add_peer_properties_column.down.sql rename to db/migrations/000025_add_peer_properties_column.down.sql diff --git a/pkg/db/migrations/000025_add_peer_properties_column.up.sql b/db/migrations/000025_add_peer_properties_column.up.sql similarity index 100% rename from pkg/db/migrations/000025_add_peer_properties_column.up.sql rename to db/migrations/000025_add_peer_properties_column.up.sql diff --git a/pkg/db/migrations/000026_add_net_errors.down.sql b/db/migrations/000026_add_net_errors.down.sql similarity index 100% rename from pkg/db/migrations/000026_add_net_errors.down.sql rename to db/migrations/000026_add_net_errors.down.sql diff --git a/pkg/db/migrations/000026_add_net_errors.up.sql b/db/migrations/000026_add_net_errors.up.sql similarity index 100% rename from pkg/db/migrations/000026_add_net_errors.up.sql rename to db/migrations/000026_add_net_errors.up.sql diff --git a/pkg/db/migrations/000027_alter_table_crawls_add_nebula_version.down.sql b/db/migrations/000027_alter_table_crawls_add_nebula_version.down.sql similarity index 100% rename from pkg/db/migrations/000027_alter_table_crawls_add_nebula_version.down.sql rename to db/migrations/000027_alter_table_crawls_add_nebula_version.down.sql diff --git a/pkg/db/migrations/000027_alter_table_crawls_add_nebula_version.up.sql b/db/migrations/000027_alter_table_crawls_add_nebula_version.up.sql similarity index 100% rename from pkg/db/migrations/000027_alter_table_crawls_add_nebula_version.up.sql rename to db/migrations/000027_alter_table_crawls_add_nebula_version.up.sql diff --git a/pkg/models/agent_versions.go b/db/models/agent_versions.go similarity index 100% rename from pkg/models/agent_versions.go rename to db/models/agent_versions.go diff --git a/pkg/models/boil_queries.go b/db/models/boil_queries.go similarity index 100% rename from pkg/models/boil_queries.go rename to db/models/boil_queries.go diff --git a/pkg/models/boil_table_names.go b/db/models/boil_table_names.go similarity index 100% rename from pkg/models/boil_table_names.go rename to db/models/boil_table_names.go diff --git a/pkg/models/boil_types.go b/db/models/boil_types.go similarity index 100% rename from pkg/models/boil_types.go rename to db/models/boil_types.go diff --git a/pkg/models/boil_view_names.go b/db/models/boil_view_names.go similarity index 100% rename from pkg/models/boil_view_names.go rename to db/models/boil_view_names.go diff --git a/pkg/models/crawl_properties.go b/db/models/crawl_properties.go similarity index 100% rename from pkg/models/crawl_properties.go rename to db/models/crawl_properties.go diff --git a/pkg/models/crawls.go b/db/models/crawls.go similarity index 100% rename from pkg/models/crawls.go rename to db/models/crawls.go diff --git a/pkg/models/ip_addresses.go b/db/models/ip_addresses.go similarity index 100% rename from pkg/models/ip_addresses.go rename to db/models/ip_addresses.go diff --git a/pkg/models/multi_addresses.go b/db/models/multi_addresses.go similarity index 100% rename from pkg/models/multi_addresses.go rename to db/models/multi_addresses.go diff --git a/pkg/models/neighbors.go b/db/models/neighbors.go similarity index 100% rename from pkg/models/neighbors.go rename to db/models/neighbors.go diff --git a/pkg/models/peer_logs.go b/db/models/peer_logs.go similarity index 100% rename from pkg/models/peer_logs.go rename to db/models/peer_logs.go diff --git a/pkg/models/peers.go b/db/models/peers.go similarity index 100% rename from pkg/models/peers.go rename to db/models/peers.go diff --git a/pkg/models/protocols.go b/db/models/protocols.go similarity index 100% rename from pkg/models/protocols.go rename to db/models/protocols.go diff --git a/pkg/models/protocols_sets.go b/db/models/protocols_sets.go similarity index 100% rename from pkg/models/protocols_sets.go rename to db/models/protocols_sets.go diff --git a/pkg/models/psql_upsert.go b/db/models/psql_upsert.go similarity index 100% rename from pkg/models/psql_upsert.go rename to db/models/psql_upsert.go diff --git a/pkg/models/sessions.go b/db/models/sessions.go similarity index 100% rename from pkg/models/sessions.go rename to db/models/sessions.go diff --git a/pkg/models/sessions_closed.go b/db/models/sessions_closed.go similarity index 100% rename from pkg/models/sessions_closed.go rename to db/models/sessions_closed.go diff --git a/pkg/models/sessions_open.go b/db/models/sessions_open.go similarity index 100% rename from pkg/models/sessions_open.go rename to db/models/sessions_open.go diff --git a/pkg/models/visits.go b/db/models/visits.go similarity index 100% rename from pkg/models/visits.go rename to db/models/visits.go diff --git a/pkg/discv5/crawler.go b/discv5/crawler.go similarity index 95% rename from pkg/discv5/crawler.go rename to discv5/crawler.go index 33a8d75c..d57baad9 100644 --- a/pkg/discv5/crawler.go +++ b/discv5/crawler.go @@ -16,12 +16,12 @@ import ( log "github.com/sirupsen/logrus" "go.uber.org/atomic" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/core" - "github.com/dennis-tra/nebula-crawler/pkg/db" - discover "github.com/dennis-tra/nebula-crawler/pkg/eth" - "github.com/dennis-tra/nebula-crawler/pkg/metrics" - "github.com/dennis-tra/nebula-crawler/pkg/models" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/core" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/db/models" + "github.com/dennis-tra/nebula-crawler/eth" + "github.com/dennis-tra/nebula-crawler/metrics" ) type CrawlerConfig struct { @@ -34,7 +34,7 @@ type Crawler struct { id string cfg *CrawlerConfig host *basichost.BasicHost - listener *discover.UDPv5 + listener *eth.UDPv5 crawledPeers int done chan struct{} } @@ -368,7 +368,7 @@ func (c *Crawler) crawlDiscV5(ctx context.Context, pi PeerInfo) chan DiscV5Resul timeouts := 0 enr, err := c.listener.RequestENR(pi.Node) - if errors.Is(err, discover.ErrTimeout) { + if errors.Is(err, eth.ErrTimeout) { timeouts += 1 } @@ -380,13 +380,13 @@ func (c *Crawler) crawlDiscV5(ctx context.Context, pi PeerInfo) chan DiscV5Resul // internally, so we won't gain much by spawning multiple parallel go // routines here. Stop the process as soon as we have received a timeout and // don't let the following calls time out as well. - for i := 0; i <= discover.NBuckets; i++ { // 15 is maximum + for i := 0; i <= eth.NBuckets; i++ { // 15 is maximum var neighbors []*enode.Node - neighbors, err = c.listener.FindNode(pi.Node, []uint{uint(discover.HashBits - i)}) + neighbors, err = c.listener.FindNode(pi.Node, []uint{uint(eth.HashBits - i)}) if err != nil { errorBits.Add(1 << i) - if errors.Is(err, discover.ErrTimeout) { + if errors.Is(err, eth.ErrTimeout) { timeouts += 1 if timeouts < 2 { continue diff --git a/pkg/discv5/crawler_test.go b/discv5/crawler_test.go similarity index 96% rename from pkg/discv5/crawler_test.go rename to discv5/crawler_test.go index 4ace8594..bfe4b250 100644 --- a/pkg/discv5/crawler_test.go +++ b/discv5/crawler_test.go @@ -6,7 +6,7 @@ import ( ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" - "github.com/dennis-tra/nebula-crawler/pkg/nebtest" + "github.com/dennis-tra/nebula-crawler/nebtest" ) func Test_ensureTCPAddr(t *testing.T) { diff --git a/pkg/discv5/dialer.go b/discv5/dialer.go similarity index 88% rename from pkg/discv5/dialer.go rename to discv5/dialer.go index 15be8d56..7ca47d63 100644 --- a/pkg/discv5/dialer.go +++ b/discv5/dialer.go @@ -6,9 +6,9 @@ import ( log "github.com/sirupsen/logrus" - "github.com/dennis-tra/nebula-crawler/pkg/core" - "github.com/dennis-tra/nebula-crawler/pkg/db" - "github.com/dennis-tra/nebula-crawler/pkg/eth" + "github.com/dennis-tra/nebula-crawler/core" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/eth" ) // Dialer encapsulates a libp2p host that dials peers. diff --git a/pkg/discv5/driver_crawler.go b/discv5/driver_crawler.go similarity index 96% rename from pkg/discv5/driver_crawler.go rename to discv5/driver_crawler.go index 0cd3df8c..1d380968 100644 --- a/pkg/discv5/driver_crawler.go +++ b/discv5/driver_crawler.go @@ -27,12 +27,12 @@ import ( ma "github.com/multiformats/go-multiaddr" log "github.com/sirupsen/logrus" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/core" - "github.com/dennis-tra/nebula-crawler/pkg/db" - "github.com/dennis-tra/nebula-crawler/pkg/eth" - "github.com/dennis-tra/nebula-crawler/pkg/models" - "github.com/dennis-tra/nebula-crawler/pkg/utils" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/core" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/db/models" + "github.com/dennis-tra/nebula-crawler/eth" + "github.com/dennis-tra/nebula-crawler/utils" ) type PeerInfo struct { diff --git a/pkg/discv5/driver_crawler_test.go b/discv5/driver_crawler_test.go similarity index 100% rename from pkg/discv5/driver_crawler_test.go rename to discv5/driver_crawler_test.go diff --git a/pkg/discv5/driver_dialer.go b/discv5/driver_dialer.go similarity index 97% rename from pkg/discv5/driver_dialer.go rename to discv5/driver_dialer.go index 22638cef..709d2960 100644 --- a/pkg/discv5/driver_dialer.go +++ b/discv5/driver_dialer.go @@ -21,9 +21,9 @@ import ( ma "github.com/multiformats/go-multiaddr" log "github.com/sirupsen/logrus" - "github.com/dennis-tra/nebula-crawler/pkg/core" - "github.com/dennis-tra/nebula-crawler/pkg/db" - "github.com/dennis-tra/nebula-crawler/pkg/eth" + "github.com/dennis-tra/nebula-crawler/core" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/eth" ) type DialDriverConfig struct { diff --git a/pkg/discv5/enr.go b/discv5/enr.go similarity index 100% rename from pkg/discv5/enr.go rename to discv5/enr.go diff --git a/pkg/discv5/enr_test.go b/discv5/enr_test.go similarity index 100% rename from pkg/discv5/enr_test.go rename to discv5/enr_test.go diff --git a/pkg/eth/common.go b/eth/common.go similarity index 100% rename from pkg/eth/common.go rename to eth/common.go diff --git a/pkg/eth/lookup.go b/eth/lookup.go similarity index 100% rename from pkg/eth/lookup.go rename to eth/lookup.go diff --git a/pkg/eth/metrics.go b/eth/metrics.go similarity index 100% rename from pkg/eth/metrics.go rename to eth/metrics.go diff --git a/pkg/eth/node.go b/eth/node.go similarity index 100% rename from pkg/eth/node.go rename to eth/node.go diff --git a/pkg/eth/ntp.go b/eth/ntp.go similarity index 100% rename from pkg/eth/ntp.go rename to eth/ntp.go diff --git a/pkg/eth/table.go b/eth/table.go similarity index 100% rename from pkg/eth/table.go rename to eth/table.go diff --git a/pkg/eth/v4_udp.go b/eth/v4_udp.go similarity index 100% rename from pkg/eth/v4_udp.go rename to eth/v4_udp.go diff --git a/pkg/eth/v4wire/v4wire.go b/eth/v4wire/v4wire.go similarity index 100% rename from pkg/eth/v4wire/v4wire.go rename to eth/v4wire/v4wire.go diff --git a/pkg/eth/v4wire/v4wire_test.go b/eth/v4wire/v4wire_test.go similarity index 100% rename from pkg/eth/v4wire/v4wire_test.go rename to eth/v4wire/v4wire_test.go diff --git a/pkg/eth/v5_talk.go b/eth/v5_talk.go similarity index 100% rename from pkg/eth/v5_talk.go rename to eth/v5_talk.go diff --git a/pkg/eth/v5_udp.go b/eth/v5_udp.go similarity index 100% rename from pkg/eth/v5_udp.go rename to eth/v5_udp.go diff --git a/pkg/eth/v5wire/crypto.go b/eth/v5wire/crypto.go similarity index 100% rename from pkg/eth/v5wire/crypto.go rename to eth/v5wire/crypto.go diff --git a/pkg/eth/v5wire/crypto_test.go b/eth/v5wire/crypto_test.go similarity index 100% rename from pkg/eth/v5wire/crypto_test.go rename to eth/v5wire/crypto_test.go diff --git a/pkg/eth/v5wire/encoding.go b/eth/v5wire/encoding.go similarity index 99% rename from pkg/eth/v5wire/encoding.go rename to eth/v5wire/encoding.go index 51089106..51c46d51 100644 --- a/pkg/eth/v5wire/encoding.go +++ b/eth/v5wire/encoding.go @@ -365,7 +365,7 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey // Create the ephemeral key. This needs to be first because the // key is part of the ID nonce signature. - var remotePubkey = new(ecdsa.PublicKey) + remotePubkey := new(ecdsa.PublicKey) if err := challenge.Node.Load((*enode.Secp256k1)(remotePubkey)); err != nil { return nil, nil, fmt.Errorf("can't find secp256k1 key for recipient") } diff --git a/pkg/eth/v5wire/encoding_test.go b/eth/v5wire/encoding_test.go similarity index 100% rename from pkg/eth/v5wire/encoding_test.go rename to eth/v5wire/encoding_test.go diff --git a/pkg/eth/v5wire/msg.go b/eth/v5wire/msg.go similarity index 100% rename from pkg/eth/v5wire/msg.go rename to eth/v5wire/msg.go diff --git a/pkg/eth/v5wire/session.go b/eth/v5wire/session.go similarity index 100% rename from pkg/eth/v5wire/session.go rename to eth/v5wire/session.go diff --git a/pkg/eth/v5wire/testdata/v5.1-ping-handshake-enr.txt b/eth/v5wire/testdata/v5.1-ping-handshake-enr.txt similarity index 100% rename from pkg/eth/v5wire/testdata/v5.1-ping-handshake-enr.txt rename to eth/v5wire/testdata/v5.1-ping-handshake-enr.txt diff --git a/pkg/eth/v5wire/testdata/v5.1-ping-handshake.txt b/eth/v5wire/testdata/v5.1-ping-handshake.txt similarity index 100% rename from pkg/eth/v5wire/testdata/v5.1-ping-handshake.txt rename to eth/v5wire/testdata/v5.1-ping-handshake.txt diff --git a/pkg/eth/v5wire/testdata/v5.1-ping-message.txt b/eth/v5wire/testdata/v5.1-ping-message.txt similarity index 100% rename from pkg/eth/v5wire/testdata/v5.1-ping-message.txt rename to eth/v5wire/testdata/v5.1-ping-message.txt diff --git a/pkg/eth/v5wire/testdata/v5.1-whoareyou.txt b/eth/v5wire/testdata/v5.1-whoareyou.txt similarity index 100% rename from pkg/eth/v5wire/testdata/v5.1-whoareyou.txt rename to eth/v5wire/testdata/v5.1-whoareyou.txt diff --git a/pkg/api/api.go b/kubo/api.go similarity index 99% rename from pkg/api/api.go rename to kubo/api.go index b9a33f7b..ea2e7d0b 100644 --- a/pkg/api/api.go +++ b/kubo/api.go @@ -1,4 +1,4 @@ -package api +package kubo import ( "context" @@ -86,7 +86,6 @@ type BucketPeer struct { } func (c *Client) RoutingTable(ctx context.Context, host string) (*RoutingTableResponse, error) { - q := url.Values{} q.Add("arg", "wan") diff --git a/pkg/libp2p/crawler.go b/libp2p/crawler.go similarity index 94% rename from pkg/libp2p/crawler.go rename to libp2p/crawler.go index b8d54f74..7768b652 100644 --- a/pkg/libp2p/crawler.go +++ b/libp2p/crawler.go @@ -11,10 +11,10 @@ import ( ma "github.com/multiformats/go-multiaddr" log "github.com/sirupsen/logrus" - "github.com/dennis-tra/nebula-crawler/pkg/api" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/core" - "github.com/dennis-tra/nebula-crawler/pkg/utils" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/core" + "github.com/dennis-tra/nebula-crawler/kubo" + "github.com/dennis-tra/nebula-crawler/utils" ) type CrawlerConfig struct { @@ -30,7 +30,7 @@ type Crawler struct { host *basichost.BasicHost pm *pb.ProtocolMessenger crawledPeers int - client *api.Client + client *kubo.Client } var _ core.Worker[PeerInfo, core.CrawlResult[PeerInfo]] = (*Crawler)(nil) diff --git a/pkg/libp2p/crawler_api.go b/libp2p/crawler_api.go similarity index 92% rename from pkg/libp2p/crawler_api.go rename to libp2p/crawler_api.go index 52a2a1ec..4372222d 100644 --- a/pkg/libp2p/crawler_api.go +++ b/libp2p/crawler_api.go @@ -9,7 +9,7 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" - "github.com/dennis-tra/nebula-crawler/pkg/api" + "github.com/dennis-tra/nebula-crawler/kubo" ) type APIResult struct { @@ -17,10 +17,10 @@ type APIResult struct { Attempted bool // The ID response object from the Kubo API - ID *api.IDResponse + ID *kubo.IDResponse // The Kubo routing table. Doesn't contain multi addresses. Don't use this to continue crawling. - RoutingTable *api.RoutingTableResponse + RoutingTable *kubo.RoutingTableResponse } func (c *Crawler) crawlAPI(ctx context.Context, pi PeerInfo) <-chan APIResult { @@ -52,12 +52,12 @@ func (c *Crawler) crawlAPI(ctx context.Context, pi PeerInfo) <-chan APIResult { // declare responses var ( - idResp *api.IDResponse - rtResp *api.RoutingTableResponse + idResp *kubo.IDResponse + rtResp *kubo.RoutingTableResponse ) // init timeout context - tCtx, cancel := context.WithTimeout(ctx, api.RequestTimeout) + tCtx, cancel := context.WithTimeout(ctx, kubo.RequestTimeout) // start both requests in parallel and stop if either fails errg := errgroup.Group{} diff --git a/pkg/libp2p/crawler_p2p.go b/libp2p/crawler_p2p.go similarity index 97% rename from pkg/libp2p/crawler_p2p.go rename to libp2p/crawler_p2p.go index 74f9a239..56464f1f 100644 --- a/pkg/libp2p/crawler_p2p.go +++ b/libp2p/crawler_p2p.go @@ -13,10 +13,10 @@ import ( "go.uber.org/atomic" "golang.org/x/sync/errgroup" - "github.com/dennis-tra/nebula-crawler/pkg/core" - "github.com/dennis-tra/nebula-crawler/pkg/db" - "github.com/dennis-tra/nebula-crawler/pkg/metrics" - "github.com/dennis-tra/nebula-crawler/pkg/utils" + "github.com/dennis-tra/nebula-crawler/core" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/metrics" + "github.com/dennis-tra/nebula-crawler/utils" ) type P2PResult struct { diff --git a/pkg/libp2p/dialer.go b/libp2p/dialer.go similarity index 94% rename from pkg/libp2p/dialer.go rename to libp2p/dialer.go index a6497033..cc8d8624 100644 --- a/pkg/libp2p/dialer.go +++ b/libp2p/dialer.go @@ -10,10 +10,10 @@ import ( "github.com/libp2p/go-libp2p/core/peer" log "github.com/sirupsen/logrus" - "github.com/dennis-tra/nebula-crawler/pkg/core" - "github.com/dennis-tra/nebula-crawler/pkg/db" - "github.com/dennis-tra/nebula-crawler/pkg/metrics" - "github.com/dennis-tra/nebula-crawler/pkg/models" + "github.com/dennis-tra/nebula-crawler/core" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/db/models" + "github.com/dennis-tra/nebula-crawler/metrics" ) // Dialer encapsulates a libp2p host that dials peers. diff --git a/pkg/libp2p/driver_crawler.go b/libp2p/driver_crawler.go similarity index 92% rename from pkg/libp2p/driver_crawler.go rename to libp2p/driver_crawler.go index 0b2aa51a..a4e1019c 100644 --- a/pkg/libp2p/driver_crawler.go +++ b/libp2p/driver_crawler.go @@ -12,12 +12,12 @@ import ( rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" ma "github.com/multiformats/go-multiaddr" - "github.com/dennis-tra/nebula-crawler/pkg/api" - "github.com/dennis-tra/nebula-crawler/pkg/config" - "github.com/dennis-tra/nebula-crawler/pkg/core" - "github.com/dennis-tra/nebula-crawler/pkg/db" - "github.com/dennis-tra/nebula-crawler/pkg/models" - "github.com/dennis-tra/nebula-crawler/pkg/utils" + "github.com/dennis-tra/nebula-crawler/config" + "github.com/dennis-tra/nebula-crawler/core" + "github.com/dennis-tra/nebula-crawler/db" + "github.com/dennis-tra/nebula-crawler/db/models" + "github.com/dennis-tra/nebula-crawler/kubo" + "github.com/dennis-tra/nebula-crawler/utils" ) type PeerInfo struct { @@ -165,7 +165,7 @@ func (d *CrawlDriver) NewWorker() (core.Worker[PeerInfo, core.CrawlResult[PeerIn host: d.host, pm: pm, cfg: d.cfg.CrawlerConfig(), - client: api.NewClient(), + client: kubo.NewClient(), } d.crawlerCount += 1 diff --git a/pkg/libp2p/driver_dialer.go b/libp2p/driver_dialer.go similarity index 97% rename from pkg/libp2p/driver_dialer.go rename to libp2p/driver_dialer.go index f181fe57..ef5518ea 100644 --- a/pkg/libp2p/driver_dialer.go +++ b/libp2p/driver_dialer.go @@ -14,8 +14,8 @@ import ( ma "github.com/multiformats/go-multiaddr" log "github.com/sirupsen/logrus" - "github.com/dennis-tra/nebula-crawler/pkg/core" - "github.com/dennis-tra/nebula-crawler/pkg/db" + "github.com/dennis-tra/nebula-crawler/core" + "github.com/dennis-tra/nebula-crawler/db" ) type DialDriverConfig struct { diff --git a/pkg/libp2p/rpc.go b/libp2p/rpc.go similarity index 100% rename from pkg/libp2p/rpc.go rename to libp2p/rpc.go diff --git a/pkg/maxmind/GeoLite2-ASN.mmdb b/maxmind/GeoLite2-ASN.mmdb similarity index 100% rename from pkg/maxmind/GeoLite2-ASN.mmdb rename to maxmind/GeoLite2-ASN.mmdb diff --git a/pkg/maxmind/GeoLite2-Country.mmdb b/maxmind/GeoLite2-Country.mmdb similarity index 100% rename from pkg/maxmind/GeoLite2-Country.mmdb rename to maxmind/GeoLite2-Country.mmdb diff --git a/pkg/maxmind/maxmind.go b/maxmind/maxmind.go similarity index 100% rename from pkg/maxmind/maxmind.go rename to maxmind/maxmind.go diff --git a/pkg/maxmind/maxmind_test.go b/maxmind/maxmind_test.go similarity index 100% rename from pkg/maxmind/maxmind_test.go rename to maxmind/maxmind_test.go diff --git a/pkg/metrics/prom.go b/metrics/prom.go similarity index 98% rename from pkg/metrics/prom.go rename to metrics/prom.go index 65fe3dd6..fbd4fc42 100644 --- a/pkg/metrics/prom.go +++ b/metrics/prom.go @@ -10,7 +10,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" - "github.com/dennis-tra/nebula-crawler/pkg/config" + "github.com/dennis-tra/nebula-crawler/config" ) var ( diff --git a/pkg/nebtest/nebtest.go b/nebtest/nebtest.go similarity index 100% rename from pkg/nebtest/nebtest.go rename to nebtest/nebtest.go diff --git a/sqlboiler.toml b/sqlboiler.toml index d7978af3..6760751c 100644 --- a/sqlboiler.toml +++ b/sqlboiler.toml @@ -1,4 +1,4 @@ -output = "pkg/models" +output = "db/models" wipe = true [psql] diff --git a/pkg/udger/udger.go b/udger/udger.go similarity index 100% rename from pkg/udger/udger.go rename to udger/udger.go diff --git a/pkg/udger/udger_test.go b/udger/udger_test.go similarity index 100% rename from pkg/udger/udger_test.go rename to udger/udger_test.go diff --git a/pkg/utils/utils.go b/utils/utils.go similarity index 100% rename from pkg/utils/utils.go rename to utils/utils.go diff --git a/pkg/utils/utils_test.go b/utils/utils_test.go similarity index 96% rename from pkg/utils/utils_test.go rename to utils/utils_test.go index c55c079c..dca67b43 100644 --- a/pkg/utils/utils_test.go +++ b/utils/utils_test.go @@ -1,13 +1,12 @@ package utils import ( + nt "github.com/dennis-tra/nebula-crawler/nebtest" "sort" "testing" ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" - - nt "github.com/dennis-tra/nebula-crawler/pkg/nebtest" ) func TestMergeMaddrs(t *testing.T) {