diff --git a/.github/workflows/crystal_build.yml b/.github/workflows/crystal_build.yml index 4eab45be..30e77a9e 100644 --- a/.github/workflows/crystal_build.yml +++ b/.github/workflows/crystal_build.yml @@ -1,8 +1,6 @@ name: Crystal Build on: - push: - branches: [ "main", "dev" ] pull_request: branches: [ "main", "dev" ] diff --git a/.github/workflows/crystal_lint.yml b/.github/workflows/crystal_lint.yml index 8442e4be..a4f8440a 100644 --- a/.github/workflows/crystal_lint.yml +++ b/.github/workflows/crystal_lint.yml @@ -1,7 +1,7 @@ name: Crystal Lint on: - push: + pull_request: branches: [ "main", "dev" ] jobs: diff --git a/.github/workflows/crystal_test.yml b/.github/workflows/crystal_test.yml index 02eca602..24088d64 100644 --- a/.github/workflows/crystal_test.yml +++ b/.github/workflows/crystal_test.yml @@ -1,8 +1,6 @@ name: Crystal Test on: - push: - branches: [ "main", "dev" ] pull_request: branches: [ "main", "dev" ] diff --git a/README.md b/README.md index 5b8a9b8b..e000d372 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ - Header - Cookie - Protocol (e.g ws) +- Details (e.g The origin of the endpoint) ### Languages and Frameworks @@ -26,6 +27,7 @@ | Crystal | Lucky | ✅ | ✅ | ✅ | ✅ | ✅ | X | | Go | Echo | ✅ | ✅ | ✅ | ✅ | ✅ | X | | Go | Gin | ✅ | ✅ | ✅ | ✅ | ✅ | X | +| Go | Fiber | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | Python | Django | ✅ | ✅ | ✅ | ✅ | ✅ | X | | Python | Flask | ✅ | ✅ | ✅ | ✅ | ✅ | X | | Python | FastAPI | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | @@ -37,7 +39,7 @@ | Java | Armeria | ✅ | ✅ | X | X | X | X | | Java | Spring | ✅ | ✅ | X | X | X | X | | Kotlin | Spring | ✅ | ✅ | X | X | X | X | -| JS | Express | ✅ | ✅ | X | X | X | X | +| JS | Express | ✅ | ✅ | ✅ | ✅ | ✅ | X | | Rust | Axum | ✅ | ✅ | X | X | X | X | | Elixir | Phoenix | ✅ | ✅ | X | X | X | ✅ | | C# | ASP.NET MVC | ✅ | X | X | X | X | X | @@ -92,13 +94,13 @@ Usage: noir Basic: -b PATH, --base-path ./app (Required) Set base path -u URL, --url http://.. Set base url for endpoints - -s SCOPE, --scope url,param Set scope for detection Output: -f FORMAT, --format json Set output format [plain/json/yaml/markdown-table/curl/httpie/oas2/oas3] -o PATH, --output out.txt Write result to file --set-pvalue VALUE Specifies the value of the identified parameter + --include-path Include file path in the plain result --no-color Disable color output --no-log Displaying only the results @@ -135,6 +137,7 @@ JSON Result ``` noir -b . -u https://testapp.internal.domains -f json ``` + ```json [ ... @@ -159,7 +162,15 @@ noir -b . -u https://testapp.internal.domains -f json } ], "protocol": "http", - "url": "https://testapp.internal.domains/comments" + "url": "https://testapp.internal.domains/comments", + "details": { + "code_paths": [ + { + "path": "app_source/testapp.cr", + "line": 3 + } + ] + } } ] ``` diff --git a/shard.yml b/shard.yml index 8de6f158..29cb8903 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: noir -version: 0.11.0 +version: 0.12.0 authors: - hahwul diff --git a/spec/functional_test/fixtures/file_based/base64.txt b/spec/functional_test/fixtures/file_based/base64.txt new file mode 100644 index 00000000..f286f6e0 --- /dev/null +++ b/spec/functional_test/fixtures/file_based/base64.txt @@ -0,0 +1,2 @@ +aHR0cHM6Ly93d3cuaGFod3VsLmNvbS90YWcvY3J5c3RhbC8= +eyJ6YXAiOiJodHRwczovL3d3dy5oYWh3dWwuY29tL3RhZy96YXAvIn0= \ No newline at end of file diff --git a/spec/functional_test/fixtures/file_based/urls.json b/spec/functional_test/fixtures/file_based/urls.json new file mode 100644 index 00000000..7816ea1d --- /dev/null +++ b/spec/functional_test/fixtures/file_based/urls.json @@ -0,0 +1,6 @@ +{ + "url": [ + "https://www.hahwul.com/tag/security/", + "https://www.hahwul.com/phoenix" + ] +} \ No newline at end of file diff --git a/spec/functional_test/fixtures/file_based/urls.txt b/spec/functional_test/fixtures/file_based/urls.txt new file mode 100644 index 00000000..9af3d949 --- /dev/null +++ b/spec/functional_test/fixtures/file_based/urls.txt @@ -0,0 +1,3 @@ +Main: https://www.hahwul.com/ +About page: https://www.hahwul.com/about +Wiki: https://www.hahwul.com/cullinan \ No newline at end of file diff --git a/spec/functional_test/fixtures/go_echo/public2/mob.txt b/spec/functional_test/fixtures/go_echo/public2/mob.txt new file mode 100644 index 00000000..e69de29b diff --git a/spec/functional_test/fixtures/go_echo/public3/coffee.txt b/spec/functional_test/fixtures/go_echo/public3/coffee.txt new file mode 100644 index 00000000..e69de29b diff --git a/spec/functional_test/fixtures/go_echo/server.go b/spec/functional_test/fixtures/go_echo/server.go index f0e6629d..2310e29a 100644 --- a/spec/functional_test/fixtures/go_echo/server.go +++ b/spec/functional_test/fixtures/go_echo/server.go @@ -26,5 +26,8 @@ func main() { return c.String(http.StatusOK, "Hello, Pet!") }) e.Static("/public", "public") + e.Static("/public", "./public2") + e.Static("/public", "/public3") + e.Logger.Fatal(e.Start(":1323")) } diff --git a/spec/functional_test/fixtures/go_fiber/go.mod b/spec/functional_test/fixtures/go_fiber/go.mod new file mode 100644 index 00000000..22094584 --- /dev/null +++ b/spec/functional_test/fixtures/go_fiber/go.mod @@ -0,0 +1,18 @@ +module github.com/hahwul/test-go-app + +go 1.20 + +require ( + github.com/andybalholm/brotli v1.0.6 // indirect + github.com/gofiber/fiber/v2 v2.51.0 // indirect + github.com/google/uuid v1.5.0 // indirect + github.com/klauspost/compress v1.17.4 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.51.0 // indirect + github.com/valyala/tcplisten v1.0.0 // indirect + golang.org/x/sys v0.15.0 // indirect +) \ No newline at end of file diff --git a/spec/functional_test/fixtures/go_fiber/public/secret.html b/spec/functional_test/fixtures/go_fiber/public/secret.html new file mode 100644 index 00000000..e69de29b diff --git a/spec/functional_test/fixtures/go_fiber/server.go b/spec/functional_test/fixtures/go_fiber/server.go new file mode 100644 index 00000000..9daa248b --- /dev/null +++ b/spec/functional_test/fixtures/go_fiber/server.go @@ -0,0 +1,34 @@ +package main + +import ( + "log" + + fiber "github.com/gofiber/fiber/v2" +) + +func main() { + app := fiber.New() + + // GET /api/register + app.Get("/info", func(c *fiber.Ctx) error { + msg := c.Query("sort") + return c.SendString(msg) // => ✋ register + }) + + app.Post("/update", func(c *fiber.Ctx) error { + msg := "Hello, World!" + c.Cookies("auth") + c.FormValue("name") + c.GetRespHeader("X-API-Key") + c.Vary("Origin") + return c.SendString(msg) // => ✋ register + }) + + app.Get("/ws", websocket.New(func(c *websocket.Conn) { + // Websocket logic + })) + + app.Static("/", "/public") + + log.Fatal(app.Listen(":3000")) +} diff --git a/spec/functional_test/fixtures/js_express/app.js b/spec/functional_test/fixtures/js_express/app.js index d3cc2cd4..7e404f9a 100644 --- a/spec/functional_test/fixtures/js_express/app.js +++ b/spec/functional_test/fixtures/js_express/app.js @@ -2,9 +2,14 @@ require('express') module.exports = function(app) { app.get('/',function(req,res){ + var userAgent = req.header('X-API-Key'); + var paramName = req.query.name; + res.render('index'); }); app.post('/upload',function(req,res){ res.render('index'); + var auth = req.cookies.auth; + const name = req.body.name; }); } \ No newline at end of file diff --git a/spec/functional_test/func_spec.cr b/spec/functional_test/func_spec.cr index a99ca731..b5b13a73 100644 --- a/spec/functional_test/func_spec.cr +++ b/spec/functional_test/func_spec.cr @@ -124,4 +124,8 @@ class FunctionalTester test_analyze end end + + def app + @app + end end diff --git a/spec/functional_test/testers/file_based_spec.cr b/spec/functional_test/testers/file_based_spec.cr new file mode 100644 index 00000000..80cc73a0 --- /dev/null +++ b/spec/functional_test/testers/file_based_spec.cr @@ -0,0 +1,19 @@ +require "../func_spec.cr" + +extected_endpoints = [ + Endpoint.new("https://www.hahwul.com/", "GET"), + Endpoint.new("https://www.hahwul.com/about", "GET"), + Endpoint.new("https://www.hahwul.com/cullinan", "GET"), + Endpoint.new("https://www.hahwul.com/phoenix", "GET"), + Endpoint.new("https://www.hahwul.com/tag/security/", "GET"), + Endpoint.new("https://www.hahwul.com/tag/crystal/", "GET"), + Endpoint.new("https://www.hahwul.com/tag/zap/", "GET"), +] + +tester = FunctionalTester.new("fixtures/file_based/", { + :techs => 0, + :endpoints => 7, +}, extected_endpoints) + +tester.app.options[:url] = "https://www.hahwul.com" +tester.test_all diff --git a/spec/functional_test/testers/go_echo_spec.cr b/spec/functional_test/testers/go_echo_spec.cr index 504be7c8..292af232 100644 --- a/spec/functional_test/testers/go_echo_spec.cr +++ b/spec/functional_test/testers/go_echo_spec.cr @@ -15,9 +15,11 @@ extected_endpoints = [ Param.new("name", "", "form"), ]), Endpoint.new("/public/secret.html", "GET"), + Endpoint.new("/public/mob.txt", "GET"), + Endpoint.new("/public/coffee.txt", "GET"), ] FunctionalTester.new("fixtures/go_echo/", { :techs => 1, - :endpoints => 5, + :endpoints => 7, }, extected_endpoints).test_all diff --git a/spec/functional_test/testers/go_fiber_spec.cr b/spec/functional_test/testers/go_fiber_spec.cr new file mode 100644 index 00000000..6898bf02 --- /dev/null +++ b/spec/functional_test/testers/go_fiber_spec.cr @@ -0,0 +1,20 @@ +require "../func_spec.cr" + +extected_endpoints = [ + Endpoint.new("/info", "GET", [ + Param.new("sort", "", "query"), + ]), + Endpoint.new("/update", "POST", [ + Param.new("name", "", "form"), + Param.new("auth", "", "cookie"), + Param.new("X-API-Key", "", "header"), + Param.new("Vary", "Origin", "header"), + ]), + Endpoint.new("/secret.html", "GET"), + Endpoint.new("/ws", "GET"), +] + +FunctionalTester.new("fixtures/go_fiber/", { + :techs => 1, + :endpoints => 4, +}, extected_endpoints).test_all diff --git a/spec/functional_test/testers/js_express_spec.cr b/spec/functional_test/testers/js_express_spec.cr index e9bcc1ab..99f21225 100644 --- a/spec/functional_test/testers/js_express_spec.cr +++ b/spec/functional_test/testers/js_express_spec.cr @@ -1,8 +1,14 @@ require "../func_spec.cr" extected_endpoints = [ - Endpoint.new("/", "GET"), - Endpoint.new("/upload", "POST"), + Endpoint.new("/", "GET", [ + Param.new("name", "", "query"), + Param.new("X-API-Key", "", "header"), + ]), + Endpoint.new("/upload", "POST", [ + Param.new("name", "", "json"), + Param.new("auth", "", "cookie"), + ]), ] FunctionalTester.new("fixtures/js_express/", { diff --git a/spec/unit_test/analyzer/analyzer_spring_spec.cr b/spec/unit_test/analyzer/analyzer_spring_spec.cr index 39cd3fa9..ac21d265 100644 --- a/spec/unit_test/analyzer/analyzer_spring_spec.cr +++ b/spec/unit_test/analyzer/analyzer_spring_spec.cr @@ -26,6 +26,15 @@ describe "mapping_to_path" do it "mapping_to_path - code style2" do instance.mapping_to_path("@GetMapping({ \"/abcd\" })").should eq(["/abcd"]) end + it "mapping_to_path - code style3" do + instance.mapping_to_path("@GetMapping(\"abcd\")").should eq(["/abcd"]) + end + it "mapping_to_path - code style4" do + instance.mapping_to_path("@GetMapping(value = \"abcd\")").should eq(["/abcd"]) + end + it "mapping_to_path - code style5" do + instance.mapping_to_path("@GetMapping({ \"abcd\" })").should eq(["/abcd"]) + end it "mapping_to_path - multiple path" do instance.mapping_to_path("@GetMapping(value={\"/abcd\", \"/efgh\"})").should eq(["/abcd", "/efgh"]) end @@ -56,6 +65,9 @@ describe "mapping_to_path" do it "mapping_to_path - requestmapping style6" do instance.mapping_to_path("@RequestMapping(\"/abcd\", produces=[MediaType.APPLICATION_JSON_VALUE])").should eq(["/abcd"]) end + it "mapping_to_path - requestmapping style7" do + instance.mapping_to_path("@GetMapping").should eq([""]) + end end describe "utils func" do diff --git a/spec/unit_test/models/analyzer_spec.cr b/spec/unit_test/models/analyzer_spec.cr index d0a9e8e3..bc699c77 100644 --- a/spec/unit_test/models/analyzer_spec.cr +++ b/spec/unit_test/models/analyzer_spec.cr @@ -1,7 +1,7 @@ require "../../../src/models/analyzer.cr" require "../../../src/options.cr" -describe "Initialize" do +describe "Initialize Analyzer" do options = default_options options[:base] = "noir" object = Analyzer.new(options) @@ -10,8 +10,23 @@ describe "Initialize" do object.url.should eq("") end - it "getter - scope" do - object.scope.should eq("url,param") + it "getter - result" do + empty = [] of Endpoint + object.result.should eq(empty) + end + + it "initialized - base_path" do + object.base_path.should eq("noir") + end +end + +describe "Initialize FileAnalyzer" do + options = default_options + options[:base] = "noir" + object = FileAnalyzer.new(options) + + it "getter - url" do + object.url.should eq("") end it "getter - result" do @@ -22,4 +37,8 @@ describe "Initialize" do it "initialized - base_path" do object.base_path.should eq("noir") end + + it "getter - hooks_count" do + object.hooks_count.should_not eq(0) + end end diff --git a/spec/unit_test/models/endpoint_spec.cr b/spec/unit_test/models/endpoint_spec.cr index ea1914c7..2dae8de6 100644 --- a/spec/unit_test/models/endpoint_spec.cr +++ b/spec/unit_test/models/endpoint_spec.cr @@ -24,4 +24,41 @@ describe "Initialize 3 arguments" do it "detect_params" do endpoint.params.should eq([Param.new("a", "b", "query")]) end + + path = "path/a/b/c" + line = 123 + path_info = PathInfo.new(path, line) + endpoint2 = Endpoint.new("/abcd", "GET", Details.new(path_info)) + it "detect_url" do + endpoint2.url.should eq("/abcd") + end + it "detect_method" do + endpoint2.method.should eq("GET") + end + it "detect_details" do + endpoint2.details.should eq(Details.new(path_info)) + endpoint2.details.code_paths[0].path.should eq(path) + endpoint2.details.code_paths[0].line.should eq(line) + end +end + +describe "Initialize 4 arguments" do + path = "path/a/b/c" + line = 123 + path_info = PathInfo.new(path, line) + endpoint = Endpoint.new("/abcd", "GET", [Param.new("a", "b", "query")], Details.new(path_info)) + it "detect_url" do + endpoint.url.should eq("/abcd") + end + it "detect_method" do + endpoint.method.should eq("GET") + end + it "detect_params" do + endpoint.params.should eq([Param.new("a", "b", "query")]) + end + it "detect_details" do + endpoint.details.should eq(Details.new(path_info)) + endpoint.details.code_paths[0].path.should eq(path) + endpoint.details.code_paths[0].line.should eq(line) + end end diff --git a/spec/unit_test/models/noir_spec.cr b/spec/unit_test/models/noir_spec.cr new file mode 100644 index 00000000..4b117634 --- /dev/null +++ b/spec/unit_test/models/noir_spec.cr @@ -0,0 +1,31 @@ +require "../../../src/models/noir.cr" +require "../../../src/options.cr" +require "../../../src/models/endpoint.cr" + +describe "Initialize" do + options = default_options + options[:base] = "noir" + runner = NoirRunner.new(options) + + it "getter - options" do + tmp_options = runner.options + tmp_options[:base].should eq(options[:base]) + end +end + +describe "Methods" do + options = default_options + options[:base] = "noir" + options[:url] = "https://www.hahwul.com" + options[:nolog] = "yes" + runner = NoirRunner.new(options) + + runner.endpoints << Endpoint.new("/abcd", "GET") + runner.endpoints << Endpoint.new("abcd", "GET") + + it "combine_url_and_endpoints" do + runner.combine_url_and_endpoints + runner.endpoints[0].url.should eq("https://www.hahwul.com/abcd") + runner.endpoints[1].url.should eq("https://www.hahwul.com/abcd") + end +end diff --git a/spec/unit_test/models/output_builder_spec.cr b/spec/unit_test/models/output_builder_spec.cr index 9c128e4b..fe35ab90 100644 --- a/spec/unit_test/models/output_builder_spec.cr +++ b/spec/unit_test/models/output_builder_spec.cr @@ -4,40 +4,41 @@ require "../../../src/options.cr" describe "Initialize" do options = default_options options[:base] = "noir" - options[:scope] = "param" + options[:format] = "json" + options[:output] = "output.json" it "OutputBuilder" do object = OutputBuilder.new options - object.scope.should eq("param") + object.output_file.should eq("output.json") end it "OutputBuilderCommon" do object = OutputBuilderCommon.new options - object.scope.should eq("param") + object.output_file.should eq("output.json") end it "OutputBuilderCurl" do object = OutputBuilderCurl.new options - object.scope.should eq("param") + object.output_file.should eq("output.json") end it "OutputBuilderHttpie" do object = OutputBuilderHttpie.new options - object.scope.should eq("param") + object.output_file.should eq("output.json") end it "OutputBuilderMarkdownTable" do object = OutputBuilderMarkdownTable.new options - object.scope.should eq("param") + object.output_file.should eq("output.json") end it "OutputBuilderOas2" do object = OutputBuilderOas2.new options - object.scope.should eq("param") + object.output_file.should eq("output.json") end it "OutputBuilderOas3" do object = OutputBuilderOas3.new options - object.scope.should eq("param") + object.output_file.should eq("output.json") end end diff --git a/spec/unit_test/utils/string_extension_spec.cr b/spec/unit_test/utils/string_extension_spec.cr new file mode 100644 index 00000000..232007a1 --- /dev/null +++ b/spec/unit_test/utils/string_extension_spec.cr @@ -0,0 +1,19 @@ +require "../../../src/utils/*" + +describe "gsub_repeatedly" do + it "basic" do + "ab//cd".gsub_repeatedly("//", "/").should eq("ab/cd") + end + + it "bulk" do + "ab//////cd".gsub_repeatedly("//", "/").should eq("ab/cd") + end + + it "origin blank case" do + "".gsub_repeatedly("//", "/").should eq("") + end + + it "pattern blank case" do + "/abcd".gsub_repeatedly("", "").should eq("/abcd") + end +end diff --git a/src/analyzer/analyzer.cr b/src/analyzer/analyzer.cr index fddf42f1..2dab9bd6 100644 --- a/src/analyzer/analyzer.cr +++ b/src/analyzer/analyzer.cr @@ -1,4 +1,5 @@ require "./analyzers/*" +require "./analyzers/file_analyzers/*" def initialize_analyzers(logger : NoirLogger) analyzers = {} of String => Proc(Hash(Symbol, String), Array(Endpoint)) @@ -11,6 +12,7 @@ def initialize_analyzers(logger : NoirLogger) analyzers["php_pure"] = ->analyzer_php_pure(Hash(Symbol, String)) analyzers["go_echo"] = ->analyzer_go_echo(Hash(Symbol, String)) analyzers["go_gin"] = ->analyzer_go_gin(Hash(Symbol, String)) + analyzers["go_fiber"] = ->analyzer_go_fiber(Hash(Symbol, String)) analyzers["python_flask"] = ->analyzer_flask(Hash(Symbol, String)) analyzers["python_fastapi"] = ->analyzer_fastapi(Hash(Symbol, String)) analyzers["python_django"] = ->analyzer_django(Hash(Symbol, String)) @@ -35,10 +37,16 @@ end def analysis_endpoints(options : Hash(Symbol, String), techs, logger : NoirLogger) result = [] of Endpoint - logger.system "Starting analysis of endpoints." + file_analyzer = FileAnalyzer.new options + logger.system "Initializing analyzers" analyzer = initialize_analyzers logger - logger.info_sub "Analysis to #{techs.size} technologies" + if options[:url] != "" + logger.info_sub "File analyzer initialized and #{file_analyzer.hooks_count} hooks loaded" + end + + logger.system "Analysis Started" + logger.info_sub "Code Analyzer: #{techs.size} in use" if (techs.includes? "java_spring") && (techs.includes? "kotlin_spring") techs.delete("kotlin_spring") @@ -54,6 +62,11 @@ def analysis_endpoints(options : Hash(Symbol, String), techs, logger : NoirLogge end end - logger.info_sub "#{result.size} endpoints found" + if options[:url] != "" + logger.info_sub "File-based Analyzer: #{file_analyzer.hooks_count} hook in use" + result = result + file_analyzer.analyze + end + + logger.info_sub "Found #{result.size} endpoints" result end diff --git a/src/analyzer/analyzers/analyzer_armeria.cr b/src/analyzer/analyzers/analyzer_armeria.cr index 5170d3f8..2a5e8850 100644 --- a/src/analyzer/analyzers/analyzer_armeria.cr +++ b/src/analyzer/analyzers/analyzer_armeria.cr @@ -11,8 +11,9 @@ class AnalyzerArmeria < Analyzer Dir.glob("#{@base_path}/**/*") do |path| next if File.directory?(path) - url = @url if File.exists?(path) && (path.ends_with?(".java") || path.ends_with?(".kt")) + details = Details.new(PathInfo.new(path)) + content = File.read(path, encoding: "utf-8", invalid: :skip) content.scan(REGEX_SERVER_CODE_BLOCK) do |server_codeblcok_match| server_codeblock = server_codeblcok_match[0] @@ -31,7 +32,7 @@ class AnalyzerArmeria < Analyzer endpoint = split_params[endpoint_param_index].strip endpoint = endpoint[1..-2] - @result << Endpoint.new("#{url}#{endpoint}", "GET") + @result << Endpoint.new("#{endpoint}", "GET", details) end server_codeblock.scan(REGEX_ROUTE_CODE) do |route_code_match| @@ -48,7 +49,7 @@ class AnalyzerArmeria < Analyzer next if endpoint[0] != '"' endpoint = endpoint[1..-2] - @result << Endpoint.new("#{url}#{endpoint}", method) + @result << Endpoint.new("#{endpoint}", method, details) end end end diff --git a/src/analyzer/analyzers/analyzer_crystal_kemal.cr b/src/analyzer/analyzers/analyzer_crystal_kemal.cr index 2e9c8dae..7914606f 100644 --- a/src/analyzer/analyzers/analyzer_crystal_kemal.cr +++ b/src/analyzer/analyzers/analyzer_crystal_kemal.cr @@ -9,9 +9,11 @@ class AnalyzerCrystalKemal < Analyzer if File.exists?(path) && File.extname(path) == ".cr" && !path.includes?("lib") File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| last_endpoint = Endpoint.new("", "") - file.each_line do |line| + file.each_line.with_index do |line, index| endpoint = line_to_endpoint(line) if endpoint.method != "" + details = Details.new(PathInfo.new(path, index + 1)) + endpoint.set_details(details) result << endpoint last_endpoint = endpoint end @@ -70,49 +72,49 @@ class AnalyzerCrystalKemal < Analyzer def line_to_endpoint(content : String) : Endpoint content.scan(/get\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "GET") + return Endpoint.new("#{match[1]}", "GET") end end content.scan(/post\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "POST") + return Endpoint.new("#{match[1]}", "POST") end end content.scan(/put\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "PUT") + return Endpoint.new("#{match[1]}", "PUT") end end content.scan(/delete\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "DELETE") + return Endpoint.new("#{match[1]}", "DELETE") end end content.scan(/patch\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "PATCH") + return Endpoint.new("#{match[1]}", "PATCH") end end content.scan(/head\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "HEAD") + return Endpoint.new("#{match[1]}", "HEAD") end end content.scan(/options\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "OPTIONS") + return Endpoint.new("#{match[1]}", "OPTIONS") end end content.scan(/ws\s+['"](.+?)['"]/) do |match| if match.size > 1 - endpoint = Endpoint.new("#{@url}#{match[1]}", "GET") + endpoint = Endpoint.new("#{match[1]}", "GET") endpoint.set_protocol("ws") return endpoint end diff --git a/src/analyzer/analyzers/analyzer_crystal_lucky.cr b/src/analyzer/analyzers/analyzer_crystal_lucky.cr index d391e18e..2e428647 100644 --- a/src/analyzer/analyzers/analyzer_crystal_lucky.cr +++ b/src/analyzer/analyzers/analyzer_crystal_lucky.cr @@ -8,7 +8,7 @@ class AnalyzerCrystalLucky < Analyzer next if File.directory?(file) real_path = "#{@base_path}/public/".gsub(/\/+/, '/') relative_path = file.sub(real_path, "") - @result << Endpoint.new("#{@url}/#{relative_path}", "GET") + @result << Endpoint.new("/#{relative_path}", "GET") end rescue e logger.debug e @@ -21,9 +21,11 @@ class AnalyzerCrystalLucky < Analyzer if File.exists?(path) && File.extname(path) == ".cr" && !path.includes?("lib") File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| last_endpoint = Endpoint.new("", "") - file.each_line do |line| + file.each_line.with_index do |line, index| endpoint = line_to_endpoint(line) if endpoint.method != "" + details = Details.new(PathInfo.new(path, index + 1)) + endpoint.set_details(details) result << endpoint last_endpoint = endpoint end @@ -87,43 +89,43 @@ class AnalyzerCrystalLucky < Analyzer def line_to_endpoint(content : String) : Endpoint content.scan(/get\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "GET") + return Endpoint.new("#{match[1]}", "GET") end end content.scan(/post\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "POST") + return Endpoint.new("#{match[1]}", "POST") end end content.scan(/put\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "PUT") + return Endpoint.new("#{match[1]}", "PUT") end end content.scan(/delete\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "DELETE") + return Endpoint.new("#{match[1]}", "DELETE") end end content.scan(/patch\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "PATCH") + return Endpoint.new("#{match[1]}", "PATCH") end end content.scan(/trace\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "TRACE") + return Endpoint.new("#{match[1]}", "TRACE") end end content.scan(/ws\s+['"](.+?)['"]/) do |match| if match.size > 1 - endpoint = Endpoint.new("#{@url}#{match[1]}", "GET") + endpoint = Endpoint.new("#{match[1]}", "GET") endpoint.set_protocol("ws") return endpoint end diff --git a/src/analyzer/analyzers/analyzer_cs_aspnet_mvc.cr b/src/analyzer/analyzers/analyzer_cs_aspnet_mvc.cr index 06753f94..f90b93bf 100644 --- a/src/analyzer/analyzers/analyzer_cs_aspnet_mvc.cr +++ b/src/analyzer/analyzers/analyzer_cs_aspnet_mvc.cr @@ -11,7 +11,7 @@ class AnalyzerCsAspNetMvc < Analyzer maproute_check = false maproute_buffer = "" - file.each_line do |line| + file.each_line.with_index do |line, index| if line.includes? ".MapRoute(" maproute_check = true maproute_buffer = line @@ -25,7 +25,8 @@ class AnalyzerCsAspNetMvc < Analyzer buffer.split(",").each do |item| if item.includes? "url:" url = item.gsub(/url:/, "").gsub(/"/, "") - @result << Endpoint.new("/#{url}", "GET") + details = Details.new(PathInfo.new(route_config_file, index + 1)) + @result << Endpoint.new("/#{url}", "GET", details) end end diff --git a/src/analyzer/analyzers/analyzer_django.cr b/src/analyzer/analyzers/analyzer_django.cr index e6ecc4b7..4826f6b4 100644 --- a/src/analyzer/analyzers/analyzer_django.cr +++ b/src/analyzer/analyzers/analyzer_django.cr @@ -38,7 +38,7 @@ class AnalyzerDjango < AnalyzerPython Dir.glob("#{@base_path}/static/**/*") do |file| next if File.directory?(file) relative_path = file.sub("#{@base_path}/static/", "") - @result << Endpoint.new("#{@url}/#{relative_path}", "GET") + @result << Endpoint.new("/#{relative_path}", "GET") end rescue e logger.debug e @@ -111,8 +111,7 @@ class AnalyzerDjango < AnalyzerPython route = route_match[1] route = route.gsub(/^\^/, "").gsub(/\$$/, "") view = route_match[2].split(",")[0] - url = "/#{@url}/#{django_urls.prefix}/#{route}".gsub(/\/+/, "/") - + url = "/#{django_urls.prefix}/#{route}".gsub(/\/+/, "/") new_django_urls = nil view.scan(REGEX_INCLUDE_URLS) do |include_pattern_match| # Detect new url configs @@ -121,15 +120,18 @@ class AnalyzerDjango < AnalyzerPython if File.exists?(new_route_path) new_django_urls = DjangoUrls.new("#{django_urls.prefix}#{route}", new_route_path, django_urls.basepath) + details = Details.new(PathInfo.new(new_route_path)) get_endpoints(new_django_urls).each do |endpoint| + endpoint.set_details(details) endpoints << endpoint end end end next if new_django_urls != nil + details = Details.new(PathInfo.new(django_urls.filepath)) if view == "" - endpoints << Endpoint.new(url, "GET") + endpoints << Endpoint.new(url, "GET", details) else dotted_as_names_split = view.split(".") @@ -149,12 +151,13 @@ class AnalyzerDjango < AnalyzerPython if filepath != "" get_endpoint_from_files(url, filepath, function_or_class_name).each do |endpoint| + endpoint.set_details(details) endpoints << endpoint end else # By default, Django allows requests with methods other than GET as well # Prevent this flow, we need to improve trace code of 'get_endpoint_from_files() - endpoints << Endpoint.new(url, "GET") + endpoints << Endpoint.new(url, "GET", details) end end end diff --git a/src/analyzer/analyzers/analyzer_elixir_phoenix.cr b/src/analyzer/analyzers/analyzer_elixir_phoenix.cr index 37755acc..52bc3c1a 100644 --- a/src/analyzer/analyzers/analyzer_elixir_phoenix.cr +++ b/src/analyzer/analyzers/analyzer_elixir_phoenix.cr @@ -8,13 +8,14 @@ class AnalyzerElixirPhoenix < Analyzer next if File.directory?(path) if File.exists?(path) && File.extname(path) == ".ex" File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| - last_endpoint = Endpoint.new("", "") - file.each_line do |line| - endpoint = line_to_endpoint(line) - if endpoint.method != "" - @result << endpoint - last_endpoint = endpoint - _ = last_endpoint + file.each_line.with_index do |line, index| + endpoints = line_to_endpoint(line) + endpoints.each do |endpoint| + if endpoint.method != "" + details = Details.new(PathInfo.new(path, index + 1)) + endpoint.set_details(details) + @result << endpoint + end end end end @@ -27,34 +28,36 @@ class AnalyzerElixirPhoenix < Analyzer @result end - def line_to_endpoint(line : String) : Endpoint + def line_to_endpoint(line : String) : Array(Endpoint) + endpoints = Array(Endpoint).new + line.scan(/get\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match| - @result << Endpoint.new("#{@url}#{match[1]}", "GET") + endpoints << Endpoint.new("#{match[1]}", "GET") end line.scan(/post\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match| - @result << Endpoint.new("#{@url}#{match[1]}", "POST") + endpoints << Endpoint.new("#{match[1]}", "POST") end line.scan(/patch\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match| - @result << Endpoint.new("#{@url}#{match[1]}", "PATCH") + endpoints << Endpoint.new("#{match[1]}", "PATCH") end line.scan(/put\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match| - @result << Endpoint.new("#{@url}#{match[1]}", "PUT") + endpoints << Endpoint.new("#{match[1]}", "PUT") end line.scan(/delete\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match| - @result << Endpoint.new("#{@url}#{match[1]}", "DELETE") + endpoints << Endpoint.new("#{match[1]}", "DELETE") end line.scan(/socket\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match| - tmp = Endpoint.new("#{@url}#{match[1]}", "GET") + tmp = Endpoint.new("#{match[1]}", "GET") tmp.set_protocol("ws") - @result << tmp + endpoints << tmp end - Endpoint.new("", "") + endpoints end end diff --git a/src/analyzer/analyzers/analyzer_example.cr b/src/analyzer/analyzers/analyzer_example.cr index 20e16bfb..230b891b 100644 --- a/src/analyzer/analyzers/analyzer_example.cr +++ b/src/analyzer/analyzers/analyzer_example.cr @@ -8,7 +8,12 @@ class AnalyzerExample < Analyzer next if File.directory?(path) if File.exists?(path) File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| - file.each_line do |_| + file.each_line.with_index do |line, index| + # For example (Add endpoint to result) + # endpoint = Endpoint.new("/", "GET") + # details = Details.new(PathInfo.new(path, index + 1)) + # endpoint.set_details(details) + # @result << endpoint end end end diff --git a/src/analyzer/analyzers/analyzer_express.cr b/src/analyzer/analyzers/analyzer_express.cr index d6f98228..650bea7d 100644 --- a/src/analyzer/analyzers/analyzer_express.cr +++ b/src/analyzer/analyzers/analyzer_express.cr @@ -8,39 +8,20 @@ class AnalyzerExpress < Analyzer next if File.directory?(path) if File.exists?(path) File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| - file.each_line do |line| - if line.includes? ".get('/" - api_path = express_get_endpoint(line) - if api_path != "" - endpoint = (url + api_path).gsub(/\/\//, "/") - result << Endpoint.new(endpoint, "GET") - end - end - if line.includes? ".post('/" - api_path = express_get_endpoint(line) - if api_path != "" - endpoint = (url + api_path).gsub(/\/\//, "/") - result << Endpoint.new(endpoint, "POST") - end - end - if line.includes? ".put('/" - api_path = express_get_endpoint(line) - if api_path != "" - result << Endpoint.new(url + api_path, "PUT") - end - end - if line.includes? ".delete('/" - api_path = express_get_endpoint(line) - if api_path != "" - endpoint = (url + api_path).gsub(/\/\//, "/") - result << Endpoint.new(endpoint, "DELETE") - end + last_endpoint = Endpoint.new("", "") + file.each_line.with_index do |line, index| + endpoint = line_to_endpoint(line) + if endpoint.method != "" + details = Details.new(PathInfo.new(path, index + 1)) + endpoint.set_details(details) + result << endpoint + last_endpoint = endpoint end - if line.includes? ".patch('/" - api_path = express_get_endpoint(line) - if api_path != "" - endpoint = (url + api_path).gsub(/\/\//, "/") - result << Endpoint.new(endpoint, "PATCH") + + param = line_to_param(line) + if param.name != "" + if last_endpoint.method != "" + last_endpoint.push_param(param) end end end @@ -63,6 +44,65 @@ class AnalyzerExpress < Analyzer api_path end + + def line_to_param(line : String) : Param + if line.includes? "req.body." + param = line.split("req.body.")[1].split(")")[0].split("}")[0].split(";")[0] + return Param.new(param, "", "json") + end + + if line.includes? "req.query." + param = line.split("req.query.")[1].split(")")[0].split("}")[0].split(";")[0] + return Param.new(param, "", "query") + end + + if line.includes? "req.cookies." + param = line.split("req.cookies.")[1].split(")")[0].split("}")[0].split(";")[0] + return Param.new(param, "", "cookie") + end + + if line.includes? "req.header(" + param = line.split("req.header(")[1].split(")")[0].gsub(/['"]/, "") + return Param.new(param, "", "header") + end + + Param.new("", "", "") + end + + def line_to_endpoint(line : String) : Endpoint + if line.includes? ".get('/" + api_path = express_get_endpoint(line) + if api_path != "" + return Endpoint.new(api_path, "GET") + end + end + if line.includes? ".post('/" + api_path = express_get_endpoint(line) + if api_path != "" + return Endpoint.new(api_path, "POST") + end + end + if line.includes? ".put('/" + api_path = express_get_endpoint(line) + if api_path != "" + return Endpoint.new(api_path, "PUT") + end + end + if line.includes? ".delete('/" + api_path = express_get_endpoint(line) + if api_path != "" + return Endpoint.new(api_path, "DELETE") + end + end + if line.includes? ".patch('/" + api_path = express_get_endpoint(line) + if api_path != "" + return Endpoint.new(api_path, "PATCH") + end + end + + Endpoint.new("", "") + end end def analyzer_express(options : Hash(Symbol, String)) diff --git a/src/analyzer/analyzers/analyzer_fastapi.cr b/src/analyzer/analyzers/analyzer_fastapi.cr index f978a884..c3b68c09 100644 --- a/src/analyzer/analyzers/analyzer_fastapi.cr +++ b/src/analyzer/analyzers/analyzer_fastapi.cr @@ -167,7 +167,8 @@ class AnalyzerFastAPI < AnalyzerPython end end - result << Endpoint.new(router_class.join(http_route_path), http_method_name, params) + details = Details.new(PathInfo.new(path, index + 1)) + result << Endpoint.new(router_class.join(http_route_path), http_method_name, params, details) end end end diff --git a/src/analyzer/analyzers/analyzer_flask.cr b/src/analyzer/analyzers/analyzer_flask.cr index 5fd99667..ac425b2a 100644 --- a/src/analyzer/analyzers/analyzer_flask.cr +++ b/src/analyzer/analyzers/analyzer_flask.cr @@ -30,7 +30,7 @@ class AnalyzerFlask < AnalyzerPython Dir.glob("#{base_path}/**/*.py") do |path| next if File.directory?(path) File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| - file.each_line do |line| + file.each_line.with_index do |line, index| # [TODO] We should be cautious about instance replace with other variable match = line.match /(#{REGEX_PYTHON_VARIABLE_NAME})\s*=\s*Flask\s*\(/ if !match.nil? @@ -73,11 +73,12 @@ class AnalyzerFlask < AnalyzerPython if !path_str_match.nil? && path_str_match.size == 2 route_path = path_str_match[1] # [TODO] Parse methods from reference views - route_url = "#{@url}#{blueprint_prefix}#{route_path}" + route_url = "#{blueprint_prefix}#{route_path}" if !route_url.starts_with? "/" route_url = "/#{route_url}" end - result << Endpoint.new(route_url, "GET") + details = Details.new(PathInfo.new(path, index + 1)) + result << Endpoint.new(route_url, "GET", details) end end end @@ -123,6 +124,8 @@ class AnalyzerFlask < AnalyzerPython codeblock_lines = codeblock_lines[1..] get_endpoints(route_path, extra_params, codeblock_lines, prefix).each do |endpoint| + details = Details.new(PathInfo.new(path, line_index + 1)) + endpoint.set_details(details) result << endpoint end end @@ -211,7 +214,7 @@ class AnalyzerFlask < AnalyzerPython prefix = "#{prefix}/" end - route_url = "#{@url}#{prefix}#{route_path}" + route_url = "#{prefix}#{route_path}" if !route_url.starts_with? "/" route_url = "/#{route_url}" end diff --git a/src/analyzer/analyzers/analyzer_go_echo.cr b/src/analyzer/analyzers/analyzer_go_echo.cr index 00fc781e..fe293dcb 100644 --- a/src/analyzer/analyzers/analyzer_go_echo.cr +++ b/src/analyzer/analyzers/analyzer_go_echo.cr @@ -10,11 +10,12 @@ class AnalyzerGoEcho < Analyzer if File.exists?(path) && File.extname(path) == ".go" File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| last_endpoint = Endpoint.new("", "") - file.each_line do |line| + file.each_line.with_index do |line, index| + details = Details.new(PathInfo.new(path, index + 1)) if line.includes?(".GET(") || line.includes?(".POST(") || line.includes?(".PUT(") || line.includes?(".DELETE(") get_route_path(line).tap do |route_path| if route_path.size > 0 - new_endpoint = Endpoint.new("#{url}#{route_path}", line.split(".")[1].split("(")[0]) + new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0], details) result << new_endpoint last_endpoint = new_endpoint end @@ -61,7 +62,7 @@ class AnalyzerGoEcho < Analyzer end public_dirs.each do |p_dir| - full_path = (base_path + "/" + p_dir["file_path"]).gsub("//", "/") + full_path = (base_path + "/" + p_dir["file_path"]).gsub_repeatedly("//", "/") Dir.glob("#{full_path}/**/*") do |path| next if File.directory?(path) if File.exists?(path) @@ -69,7 +70,8 @@ class AnalyzerGoEcho < Analyzer p_dir["static_path"] = p_dir["static_path"][0..-2] end - result << Endpoint.new("#{url}#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET") + details = Details.new(PathInfo.new(path)) + result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET", details) end end end diff --git a/src/analyzer/analyzers/analyzer_go_fiber.cr b/src/analyzer/analyzers/analyzer_go_fiber.cr new file mode 100644 index 00000000..f133ef2f --- /dev/null +++ b/src/analyzer/analyzers/analyzer_go_fiber.cr @@ -0,0 +1,157 @@ +require "../../models/analyzer" + +class AnalyzerGoFiber < Analyzer + def analyze + # Source Analysis + public_dirs = [] of (Hash(String, String)) + begin + Dir.glob("#{base_path}/**/*") do |path| + next if File.directory?(path) + if File.exists?(path) && File.extname(path) == ".go" + File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| + last_endpoint = Endpoint.new("", "") + file.each_line.with_index do |line, index| + details = Details.new(PathInfo.new(path, index + 1)) + if line.includes?(".Get(") || line.includes?(".Post(") || line.includes?(".Put(") || line.includes?(".Delete(") + get_route_path(line).tap do |route_path| + if route_path.size > 0 + new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0].upcase, details) + if line.includes?("websocket.New(") + new_endpoint.set_protocol("ws") + end + result << new_endpoint + last_endpoint = new_endpoint + end + end + end + + if line.includes?(".Query(") || line.includes?(".FormValue(") + get_param(line).tap do |param| + if param.name.size > 0 && last_endpoint.method != "" + last_endpoint.params << param + end + end + end + + if line.includes?("Static(") + get_static_path(line).tap do |static_path| + if static_path.size > 0 + public_dirs << static_path + end + end + end + + if line.includes?("GetRespHeader(") + match = line.match(/GetRespHeader\(\"(.*)\"\)/) + if match + header_name = match[1] + last_endpoint.params << Param.new(header_name, "", "header") + end + end + + if line.includes?("Vary(") + match = line.match(/Vary\(\"(.*)\"\)/) + if match + header_value = match[1] + last_endpoint.params << Param.new("Vary", header_value, "header") + end + end + + if line.includes?("Cookies(") + match = line.match(/Cookies\(\"(.*)\"\)/) + if match + cookie_name = match[1] + last_endpoint.params << Param.new(cookie_name, "", "cookie") + end + end + end + end + end + end + rescue e + logger.debug e + end + + public_dirs.each do |p_dir| + full_path = (base_path + "/" + p_dir["file_path"]).gsub_repeatedly("//", "/") + Dir.glob("#{full_path}/**/*") do |path| + next if File.directory?(path) + if File.exists?(path) + if p_dir["static_path"].ends_with?("/") + p_dir["static_path"] = p_dir["static_path"][0..-2] + end + + details = Details.new(PathInfo.new(path)) + result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET", details) + end + end + end + + Fiber.yield + + result + end + + def get_param(line : String) : Param + param_type = "json" + if line.includes?("Query") + param_type = "query" + end + if line.includes?("FormValue") + param_type = "form" + end + + first = line.strip.split("(") + if first.size > 1 + second = first[1].split(")") + if second.size > 1 + param_name = second[0].gsub("\"", "") + rtn = Param.new(param_name, "", param_type) + + return rtn + end + end + + Param.new("", "", "") + end + + def get_static_path(line : String) : Hash(String, String) + first = line.strip.split("(") + if first.size > 1 + second = first[1].split(",") + if second.size > 1 + static_path = second[0].gsub("\"", "") + file_path = second[1].gsub("\"", "").gsub(" ", "").gsub(")", "").gsub_repeatedly("//", "/") + rtn = { + "static_path" => static_path, + "file_path" => file_path, + } + + return rtn + end + end + + { + "static_path" => "", + "file_path" => "", + } + end + + def get_route_path(line : String) : String + first = line.strip.split("(") + if first.size > 1 + second = first[1].split(",") + if second.size > 1 + route_path = second[0].gsub("\"", "") + return route_path + end + end + + "" + end +end + +def analyzer_go_fiber(options : Hash(Symbol, String)) + instance = AnalyzerGoFiber.new(options) + instance.analyze +end diff --git a/src/analyzer/analyzers/analyzer_go_gin.cr b/src/analyzer/analyzers/analyzer_go_gin.cr index 683638fd..c462a95a 100644 --- a/src/analyzer/analyzers/analyzer_go_gin.cr +++ b/src/analyzer/analyzers/analyzer_go_gin.cr @@ -10,11 +10,12 @@ class AnalyzerGoGin < Analyzer if File.exists?(path) && File.extname(path) == ".go" File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| last_endpoint = Endpoint.new("", "") - file.each_line do |line| + file.each_line.with_index do |line, index| + details = Details.new(PathInfo.new(path, index + 1)) if line.includes?(".GET(") || line.includes?(".POST(") || line.includes?(".PUT(") || line.includes?(".DELETE(") get_route_path(line).tap do |route_path| if route_path.size > 0 - new_endpoint = Endpoint.new("#{url}#{route_path}", line.split(".")[1].split("(")[0]) + new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0], details) result << new_endpoint last_endpoint = new_endpoint end @@ -55,7 +56,7 @@ class AnalyzerGoGin < Analyzer end public_dirs.each do |p_dir| - full_path = (base_path + "/" + p_dir["file_path"]).gsub("//", "/") + full_path = (base_path + "/" + p_dir["file_path"]).gsub_repeatedly("//", "/") Dir.glob("#{full_path}/**/*") do |path| next if File.directory?(path) if File.exists?(path) @@ -63,7 +64,8 @@ class AnalyzerGoGin < Analyzer p_dir["static_path"] = p_dir["static_path"][0..-2] end - result << Endpoint.new("#{url}#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET") + details = Details.new(PathInfo.new(path)) + result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET", details) end end end diff --git a/src/analyzer/analyzers/analyzer_jsp.cr b/src/analyzer/analyzers/analyzer_jsp.cr index b40086a1..676bc1a2 100644 --- a/src/analyzer/analyzers/analyzer_jsp.cr +++ b/src/analyzer/analyzers/analyzer_jsp.cr @@ -33,7 +33,8 @@ class AnalyzerJsp < Analyzer rescue next end - result << Endpoint.new("#{url}/#{relative_path}", "GET", params_query) + details = Details.new(PathInfo.new(path)) + result << Endpoint.new("/#{relative_path}", "GET", params_query, details) end end end diff --git a/src/analyzer/analyzers/analyzer_oas2.cr b/src/analyzer/analyzers/analyzer_oas2.cr index fb64bdb9..16a1071b 100644 --- a/src/analyzer/analyzers/analyzer_oas2.cr +++ b/src/analyzer/analyzers/analyzer_oas2.cr @@ -9,12 +9,13 @@ class AnalyzerOAS2 < Analyzer if swagger_jsons.is_a?(Array(String)) swagger_jsons.each do |swagger_json| if File.exists?(swagger_json) + details = Details.new(PathInfo.new(swagger_json)) content = File.read(swagger_json, encoding: "utf-8", invalid: :skip) json_obj = JSON.parse(content) - base_path = @url + base_path = "" begin if json_obj["basePath"].to_s != "" - base_path = base_path + json_obj["basePath"].to_s + base_path = json_obj["basePath"].to_s end rescue e @logger.debug "Exception of #{swagger_json}/basePath" @@ -44,9 +45,9 @@ class AnalyzerOAS2 < Analyzer params << param end end - @result << Endpoint.new(base_path + path, method.upcase, params) + @result << Endpoint.new(base_path + path, method.upcase, params, details) else - @result << Endpoint.new(base_path + path, method.upcase) + @result << Endpoint.new(base_path + path, method.upcase, details) end rescue e @logger.debug "Exception of #{swagger_json}/paths/path/method" @@ -67,12 +68,13 @@ class AnalyzerOAS2 < Analyzer if swagger_yamls.is_a?(Array(String)) swagger_yamls.each do |swagger_yaml| if File.exists?(swagger_yaml) + details = Details.new(PathInfo.new(swagger_yaml)) content = File.read(swagger_yaml, encoding: "utf-8", invalid: :skip) yaml_obj = YAML.parse(content) - base_path = @url + base_path = "" begin if yaml_obj["basePath"].to_s != "" - base_path = base_path + yaml_obj["basePath"].to_s + base_path = yaml_obj["basePath"].to_s end rescue e @logger.debug "Exception of #{swagger_yaml}/basePath" @@ -102,9 +104,9 @@ class AnalyzerOAS2 < Analyzer params << param end end - @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, params) + @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, params, details) else - @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase) + @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, details) end rescue e @logger.debug "Exception of #{swagger_yaml}/paths/path/method" diff --git a/src/analyzer/analyzers/analyzer_oas3.cr b/src/analyzer/analyzers/analyzer_oas3.cr index 7725d3dd..d7d153ed 100644 --- a/src/analyzer/analyzers/analyzer_oas3.cr +++ b/src/analyzer/analyzers/analyzer_oas3.cr @@ -26,6 +26,7 @@ class AnalyzerOAS3 < Analyzer if oas3_jsons.is_a?(Array(String)) oas3_jsons.each do |oas3_json| if File.exists?(oas3_json) + details = Details.new(PathInfo.new(oas3_json)) content = File.read(oas3_json, encoding: "utf-8", invalid: :skip) json_obj = JSON.parse(content) @@ -76,11 +77,11 @@ class AnalyzerOAS3 < Analyzer end if params.size > 0 - @result << Endpoint.new(base_path + path, method.upcase, params) + @result << Endpoint.new(base_path + path, method.upcase, params, details) elsif params.size > 0 - @result << Endpoint.new(base_path + path, method.upcase, params) + @result << Endpoint.new(base_path + path, method.upcase, params, details) else - @result << Endpoint.new(base_path + path, method.upcase) + @result << Endpoint.new(base_path + path, method.upcase, details) end rescue e @logger.debug "Exception of #{oas3_json}/paths/endpoint" @@ -98,6 +99,7 @@ class AnalyzerOAS3 < Analyzer if oas3_yamls.is_a?(Array(String)) oas3_yamls.each do |oas3_yaml| if File.exists?(oas3_yaml) + details = Details.new(PathInfo.new(oas3_yaml)) content = File.read(oas3_yaml, encoding: "utf-8", invalid: :skip) yaml_obj = YAML.parse(content) @@ -148,11 +150,11 @@ class AnalyzerOAS3 < Analyzer end if params.size > 0 - @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, params) + @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, params, details) elsif params.size > 0 - @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, params) + @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, params, details) else - @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase) + @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, details) end end rescue e diff --git a/src/analyzer/analyzers/analyzer_php_pure.cr b/src/analyzer/analyzers/analyzer_php_pure.cr index 4de753f3..c6298a03 100644 --- a/src/analyzer/analyzers/analyzer_php_pure.cr +++ b/src/analyzer/analyzers/analyzer_php_pure.cr @@ -45,10 +45,12 @@ class AnalyzerPhpPure < Analyzer rescue next end + + details = Details.new(PathInfo.new(path)) methods.each do |method| - result << Endpoint.new("#{url}/#{relative_path}", method, params_body) + result << Endpoint.new("/#{relative_path}", method, params_body, details) end - result << Endpoint.new("#{url}/#{relative_path}", "GET", params_query) + result << Endpoint.new("/#{relative_path}", "GET", params_query, details) end end end diff --git a/src/analyzer/analyzers/analyzer_raml.cr b/src/analyzer/analyzers/analyzer_raml.cr index e84864cc..f9ac8f3b 100644 --- a/src/analyzer/analyzers/analyzer_raml.cr +++ b/src/analyzer/analyzers/analyzer_raml.cr @@ -8,6 +8,8 @@ class AnalyzerRAML < Analyzer if raml_specs.is_a?(Array(String)) raml_specs.each do |raml_spec| if File.exists?(raml_spec) + details = Details.new(PathInfo.new(raml_spec)) + content = File.read(raml_spec, encoding: "utf-8", invalid: :skip) yaml_obj = YAML.parse(content) yaml_obj.as_h.each do |path, path_obj| @@ -45,7 +47,7 @@ class AnalyzerRAML < Analyzer end end - @result << Endpoint.new(path.to_s, method.to_s.upcase, params) + @result << Endpoint.new(path.to_s, method.to_s.upcase, params, details) rescue e @logger.debug "Exception of #{raml_spec}/paths/#{path}/#{method}" @logger.debug_sub e diff --git a/src/analyzer/analyzers/analyzer_ruby_hanami.cr b/src/analyzer/analyzers/analyzer_ruby_hanami.cr index a8930d21..25a07c5a 100644 --- a/src/analyzer/analyzers/analyzer_ruby_hanami.cr +++ b/src/analyzer/analyzers/analyzer_ruby_hanami.cr @@ -3,11 +3,13 @@ require "../../models/analyzer" class AnalyzerRubyHanami < Analyzer def analyze # Config Analysis - if File.exists?("#{@base_path}/config/routes.rb") - File.open("#{@base_path}/config/routes.rb", "r", encoding: "utf-8", invalid: :skip) do |file| + path = "#{@base_path}/config/routes.rb" + if File.exists?(path) + File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| last_endpoint = Endpoint.new("", "") - file.each_line do |line| - endpoint = line_to_endpoint(line) + file.each_line.with_index do |line, index| + details = Details.new(PathInfo.new(path, index + 1)) + endpoint = line_to_endpoint(line, details) if endpoint.method != "" @result << endpoint last_endpoint = endpoint @@ -20,46 +22,46 @@ class AnalyzerRubyHanami < Analyzer @result end - def line_to_endpoint(content : String) : Endpoint + def line_to_endpoint(content : String, details : Details) : Endpoint content.scan(/get\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "GET") + return Endpoint.new("#{match[1]}", "GET", details) end end content.scan(/post\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "POST") + return Endpoint.new("#{match[1]}", "POST", details) end end content.scan(/put\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "PUT") + return Endpoint.new("#{match[1]}", "PUT", details) end end content.scan(/delete\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "DELETE") + return Endpoint.new("#{match[1]}", "DELETE", details) end end content.scan(/patch\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "PATCH") + return Endpoint.new("#{match[1]}", "PATCH", details) end end content.scan(/head\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "HEAD") + return Endpoint.new("#{match[1]}", "HEAD", details) end end content.scan(/options\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "OPTIONS") + return Endpoint.new("#{match[1]}", "OPTIONS", details) end end diff --git a/src/analyzer/analyzers/analyzer_ruby_rails.cr b/src/analyzer/analyzers/analyzer_ruby_rails.cr index 099e1442..ea614c76 100644 --- a/src/analyzer/analyzers/analyzer_ruby_rails.cr +++ b/src/analyzer/analyzers/analyzer_ruby_rails.cr @@ -8,7 +8,8 @@ class AnalyzerRubyRails < Analyzer next if File.directory?(file) real_path = "#{@base_path}/public/".gsub(/\/+/, '/') relative_path = file.sub(real_path, "") - @result << Endpoint.new("#{@url}/#{relative_path}", "GET") + details = Details.new(PathInfo.new(file)) + @result << Endpoint.new("/#{relative_path}", "GET", details) end rescue e logger.debug e @@ -31,20 +32,21 @@ class AnalyzerRubyRails < Analyzer end end + details = Details.new(PathInfo.new("#{@base_path}/config/routes.rb")) line.scan(/get\s+['"](.+?)['"]/) do |match| - @result << Endpoint.new("#{@url}#{match[1]}", "GET") + @result << Endpoint.new("#{match[1]}", "GET", details) end line.scan(/post\s+['"](.+?)['"]/) do |match| - @result << Endpoint.new("#{@url}#{match[1]}", "POST") + @result << Endpoint.new("#{match[1]}", "POST", details) end line.scan(/put\s+['"](.+?)['"]/) do |match| - @result << Endpoint.new("#{@url}#{match[1]}", "PUT") + @result << Endpoint.new("#{match[1]}", "PUT", details) end line.scan(/delete\s+['"](.+?)['"]/) do |match| - @result << Endpoint.new("#{@url}#{match[1]}", "DELETE") + @result << Endpoint.new("#{match[1]}", "DELETE", details) end line.scan(/patch\s+['"](.+?)['"]/) do |match| - @result << Endpoint.new("#{@url}#{match[1]}", "PATCH") + @result << Endpoint.new("#{match[1]}", "PATCH", details) end end end @@ -188,6 +190,7 @@ class AnalyzerRubyRails < Analyzer end end + details = Details.new(PathInfo.new(path)) methods.each do |method| if method == "GET/INDEX" if params_method.has_key? "index" @@ -200,7 +203,7 @@ class AnalyzerRubyRails < Analyzer index_params ||= [] of Param deduplication_params_query ||= [] of Param last_params = index_params + deduplication_params_query - @result << Endpoint.new("#{@url}/#{resource}", "GET", last_params) + @result << Endpoint.new("/#{resource}", "GET", last_params, details) elsif method == "GET/SHOW" if params_method.has_key? "show" show_params = [] of Param @@ -211,7 +214,7 @@ class AnalyzerRubyRails < Analyzer show_params ||= [] of Param deduplication_params_query ||= [] of Param last_params = show_params + deduplication_params_query - @result << Endpoint.new("#{@url}/#{resource}/1", "GET", last_params) + @result << Endpoint.new("/#{resource}/1", "GET", last_params, details) else if method == "POST" if params_method.has_key? "create" @@ -223,7 +226,7 @@ class AnalyzerRubyRails < Analyzer create_params ||= [] of Param params_body ||= [] of Param last_params = create_params + params_body - @result << Endpoint.new("#{@url}/#{resource}", method, last_params) + @result << Endpoint.new("/#{resource}", method, last_params, details) elsif method == "DELETE" params_delete = [] of Param if params_method.has_key? "delete" @@ -231,7 +234,7 @@ class AnalyzerRubyRails < Analyzer params_delete << param end end - @result << Endpoint.new("#{@url}/#{resource}/1", method, params_delete) + @result << Endpoint.new("/#{resource}/1", method, params_delete, details) else if params_method.has_key? "update" update_params = [] of Param @@ -242,7 +245,7 @@ class AnalyzerRubyRails < Analyzer update_params ||= [] of Param params_body ||= [] of Param last_params = update_params + params_body - @result << Endpoint.new("#{@url}/#{resource}/1", method, last_params) + @result << Endpoint.new("/#{resource}/1", method, last_params, details) end end end diff --git a/src/analyzer/analyzers/analyzer_ruby_sinatra.cr b/src/analyzer/analyzers/analyzer_ruby_sinatra.cr index 4f3a5133..a7572a73 100644 --- a/src/analyzer/analyzers/analyzer_ruby_sinatra.cr +++ b/src/analyzer/analyzers/analyzer_ruby_sinatra.cr @@ -9,9 +9,11 @@ class AnalyzerRubySinatra < Analyzer if File.exists?(path) File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| last_endpoint = Endpoint.new("", "") - file.each_line do |line| + file.each_line.with_index do |line, index| endpoint = line_to_endpoint(line) if endpoint.method != "" + details = Details.new(PathInfo.new(path, index + 1)) + endpoint.set_details(details) @result << endpoint last_endpoint = endpoint end @@ -65,43 +67,43 @@ class AnalyzerRubySinatra < Analyzer def line_to_endpoint(content : String) : Endpoint content.scan(/get\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "GET") + return Endpoint.new("#{match[1]}", "GET") end end content.scan(/post\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "POST") + return Endpoint.new("#{match[1]}", "POST") end end content.scan(/put\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "PUT") + return Endpoint.new("#{match[1]}", "PUT") end end content.scan(/delete\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "DELETE") + return Endpoint.new("#{match[1]}", "DELETE") end end content.scan(/patch\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "PATCH") + return Endpoint.new("#{match[1]}", "PATCH") end end content.scan(/head\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "HEAD") + return Endpoint.new("#{match[1]}", "HEAD") end end content.scan(/options\s+['"](.+?)['"]/) do |match| if match.size > 1 - return Endpoint.new("#{@url}#{match[1]}", "OPTIONS") + return Endpoint.new("#{match[1]}", "OPTIONS") end end diff --git a/src/analyzer/analyzers/analyzer_rust_axum.cr b/src/analyzer/analyzers/analyzer_rust_axum.cr index 7fac3ad5..0b682beb 100644 --- a/src/analyzer/analyzers/analyzer_rust_axum.cr +++ b/src/analyzer/analyzers/analyzer_rust_axum.cr @@ -11,14 +11,15 @@ class AnalyzerRustAxum < Analyzer if File.exists?(path) && File.extname(path) == ".rs" File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| - file.each_line do |line| + file.each_line.with_index do |line, index| if line.includes? ".route(" match = line.match(pattern) if match begin route_argument = match[1] callback_argument = match[2] - result << Endpoint.new("#{@url}#{route_argument}", callback_to_method(callback_argument)) + details = Details.new(PathInfo.new(path, index + 1)) + result << Endpoint.new("#{route_argument}", callback_to_method(callback_argument), details) rescue end end diff --git a/src/analyzer/analyzers/analyzer_spring.cr b/src/analyzer/analyzers/analyzer_spring.cr index 5ee51d85..4e53ad09 100644 --- a/src/analyzer/analyzers/analyzer_spring.cr +++ b/src/analyzer/analyzers/analyzer_spring.cr @@ -11,13 +11,14 @@ class AnalyzerSpring < Analyzer Dir.glob("#{@base_path}/**/*") do |path| next if File.directory?(path) - url = @url + url = "" if File.exists?(path) && (path.ends_with?(".java") || path.ends_with?(".kt")) content = File.read(path, encoding: "utf-8", invalid: :skip) # Spring MVC has_class_been_imported = false - content.each_line do |line| + content.each_line.with_index do |line, index| + details = Details.new(PathInfo.new(path, index + 1)) if has_class_been_imported == false && REGEX_CLASS_DEFINITION.match(line) has_class_been_imported = true end @@ -34,13 +35,13 @@ class AnalyzerSpring < Analyzer class_mapping_url = class_mapping_url[0..-2] end - url = "#{@url}#{class_mapping_url}" + url = "#{class_mapping_url}" else mapping_paths.each do |mapping_path| if line.includes? "RequestMethod" define_requestmapping_handlers(["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS", "TRACE"]) else - @result << Endpoint.new("#{url}#{mapping_path}", "GET") + @result << Endpoint.new("#{url}#{mapping_path}", "GET", details) end end end @@ -49,31 +50,31 @@ class AnalyzerSpring < Analyzer if line.includes? "PostMapping" mapping_paths = mapping_to_path(line) mapping_paths.each do |mapping_path| - @result << Endpoint.new("#{url}#{mapping_path}", "POST") + @result << Endpoint.new("#{url}#{mapping_path}", "POST", details) end end if line.includes? "PutMapping" mapping_paths = mapping_to_path(line) mapping_paths.each do |mapping_path| - @result << Endpoint.new("#{url}#{mapping_path}", "PUT") + @result << Endpoint.new("#{url}#{mapping_path}", "PUT", details) end end if line.includes? "DeleteMapping" mapping_paths = mapping_to_path(line) mapping_paths.each do |mapping_path| - @result << Endpoint.new("#{url}#{mapping_path}", "DELETE") + @result << Endpoint.new("#{url}#{mapping_path}", "DELETE", details) end end if line.includes? "PatchMapping" mapping_paths = mapping_to_path(line) mapping_paths.each do |mapping_path| - @result << Endpoint.new("#{url}#{mapping_path}", "PATCH") + @result << Endpoint.new("#{url}#{mapping_path}", "PATCH", details) end end if line.includes? "GetMapping" mapping_paths = mapping_to_path(line) mapping_paths.each do |mapping_path| - @result << Endpoint.new("#{url}#{mapping_path}", "GET") + @result << Endpoint.new("#{url}#{mapping_path}", "GET", details) end end end @@ -85,7 +86,8 @@ class AnalyzerSpring < Analyzer next if match.size != 4 method = match[2] endpoint = match[3].gsub(/\n/, "") - @result << Endpoint.new("#{url}#{endpoint}", method) + details = Details.new(PathInfo.new(path)) + @result << Endpoint.new("#{url}#{endpoint}", method, details) end end end @@ -98,49 +100,79 @@ class AnalyzerSpring < Analyzer @result end - def mapping_to_path(content : String) + def mapping_to_path(line : String) + unless line.includes? "(" + # no path + return [""] + end + paths = Array(String).new + splited_line = line.strip.split("(") + if splited_line.size > 1 && splited_line[1].includes? ")" + params = splited_line[1].split(")")[0] + params = params.gsub(/\s/, "") # remove space + if params.size > 0 + path = nil + # value parameter + if params.includes? "value=" + value = params.split("value=")[1] + if value.size > 0 + if value[0] == '"' + path = value.split("\"")[1] + elsif value[0] == '{' && value.includes? "}" + path = value[1..].split("}")[0] + end + end + end - splited_line = content.strip.split("(") - if splited_line.size > 1 - line = splited_line[1].gsub(/"|\)| /, "").gsub(/\s/, "").strip - if line.size > 0 - if line[0].to_s == "/" - attribute_index = line.index(/,(\w)+=/) - if !attribute_index.nil? - attribute_index -= 1 - line = line[0..attribute_index] + # first parameter + if path.nil? + if params[0] == '"' + path = params.split("\"")[1] + elsif params[0] == '{' && params.includes? "}" + path = params[1..].split("}")[0] end + end - paths << line + # extract path + if path.nil? + # can't find path + paths << "" else - if is_bracket(line) - line = line.gsub(/\{|\}/, "") - end - if line.size > 0 && line[0].to_s == "/" - paths << line - else - value_flag = false - line = comma_in_bracket(line) - line.split(",").each do |comma_line| - if comma_line.to_s.includes? "value=" - tmp = comma_line.split("=") - tmp[1].gsub(/"|\)/, "").strip.split("_BRACKET_COMMA_").each do |path| - paths << "#{path.strip.gsub("\\", "").gsub(";", "")}" - value_flag = true + if path.size > 0 && path[0] == '"' && path.includes? "," + # multiple path + path.split(",").each do |each_path| + if each_path.size > 0 + if each_path[0] == '"' + paths << each_path[1..-2] + else + paths << "" end end end - if value_flag == false - paths << "" + else + # single path + if path.size > 0 && path[0] == '"' + path = path.split("\"")[1] end + + paths << path end end else + # no path paths << "" end - else - paths << "" + end + + # append slash + (0..paths.size - 1).each do |i| + path = paths[i] + if path.size > 0 && !path.starts_with? "/" + path = "/" + path + end + + paths[i] = path end paths diff --git a/src/analyzer/analyzers/file_analyzers/base64.cr b/src/analyzer/analyzers/file_analyzers/base64.cr new file mode 100644 index 00000000..7028d832 --- /dev/null +++ b/src/analyzer/analyzers/file_analyzers/base64.cr @@ -0,0 +1,30 @@ +require "base64" +require "../../../models/analyzer" +require "../../../models/endpoint" + +FileAnalyzer.add_hook(->(path : String, url : String) : Array(Endpoint) { + results = [] of Endpoint + + begin + File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| + file.each_line.with_index do |line, index| + # Check base64 encoded strings + base64_match = line.match(/([A-Za-z0-9+\/]{20,}={0,2})/) + if base64_match + decoded = Base64.decode_string(base64_match[1]) + url_match = decoded.match(/(https?:\/\/[^\s"]+)/) + if url_match + parsed_url = URI.parse(url_match[1]) + if parsed_url.to_s.includes? url + details = Details.new(PathInfo.new(path, index + 1)) + results << Endpoint.new(parsed_url.path, "GET", details) + end + end + end + end + end + rescue + end + + results +}) diff --git a/src/analyzer/analyzers/file_analyzers/string.cr b/src/analyzer/analyzers/file_analyzers/string.cr new file mode 100644 index 00000000..a4cbbbee --- /dev/null +++ b/src/analyzer/analyzers/file_analyzers/string.cr @@ -0,0 +1,24 @@ +require "../../../models/analyzer" +require "../../../models/endpoint" + +FileAnalyzer.add_hook(->(path : String, url : String) : Array(Endpoint) { + results = [] of Endpoint + + begin + File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| + file.each_line.with_index do |line, index| + url_match = line.match(/(https?:\/\/[^\s"]+)/) + if url_match + parsed_url = URI.parse(url_match[1]) + if parsed_url.to_s.includes? url + details = Details.new(PathInfo.new(path, index + 1)) + results << Endpoint.new(parsed_url.path, "GET", details) + end + end + end + end + rescue + end + + results +}) diff --git a/src/banner.cr b/src/banner.cr index ba2d9250..0b5e37f1 100644 --- a/src/banner.cr +++ b/src/banner.cr @@ -1,12 +1,11 @@ def banner content = <<-CONTENT - .__ - ____ ____ |__|______ - / \\ / _ \\| \\_ __ \\ -| | ( <♠️> ) || | \\/ -|___| /\\____/|__||__| - \\/ v#{Noir::VERSION} +░█▄─░█ ░█▀▀▀█ ▀█▀ ░█▀▀█ +░█░█░█ ░█──░█ ░█─ ░█▄▄▀ +░█──▀█ ░█▄▄▄█ ▄█▄ ░█─░█ {v#{Noir::VERSION}} + CONTENT + STDERR.puts "" STDERR.puts content STDERR.puts "" end diff --git a/src/detector/detector.cr b/src/detector/detector.cr index 26cf437f..14c3e99f 100644 --- a/src/detector/detector.cr +++ b/src/detector/detector.cr @@ -17,7 +17,7 @@ def detect_techs(base_path : String, options : Hash(Symbol, String), logger : No DetectorPhpPure, DetectorPythonDjango, DetectorPythonFlask, DetectorPythonFastAPI, DetectorRubyRails, DetectorRubySinatra, DetectorRubyHanami, DetectorOas2, DetectorOas3, DetectorRAML, DetectorGoGin, DetectorKotlinSpring, DetectorJavaArmeria, DetectorCSharpAspNetMvc, - DetectorRustAxum, DetectorElixirPhoenix, + DetectorRustAxum, DetectorElixirPhoenix, DetectorGoFiber, ]) channel = Channel(String).new diff --git a/src/detector/detectors/go_fiber.cr b/src/detector/detectors/go_fiber.cr new file mode 100644 index 00000000..62863251 --- /dev/null +++ b/src/detector/detectors/go_fiber.cr @@ -0,0 +1,15 @@ +require "../../models/detector" + +class DetectorGoFiber < Detector + def detect(filename : String, file_contents : String) : Bool + if (filename.includes? "go.mod") && (file_contents.includes? "github.com/gofiber/fiber") + true + else + false + end + end + + def set_name + @name = "go_fiber" + end +end diff --git a/src/models/analyzer.cr b/src/models/analyzer.cr index a4dd47fc..196a8e9b 100644 --- a/src/models/analyzer.cr +++ b/src/models/analyzer.cr @@ -7,21 +7,21 @@ class Analyzer @endpoint_references : Array(EndpointReference) @base_path : String @url : String - @scope : String @logger : NoirLogger @is_debug : Bool @is_color : Bool @is_log : Bool + @options : Hash(Symbol, String) def initialize(options : Hash(Symbol, String)) @base_path = options[:base] @url = options[:url] @result = [] of Endpoint @endpoint_references = [] of EndpointReference - @scope = options[:scope] @is_debug = str_to_bool(options[:debug]) @is_color = str_to_bool(options[:color]) @is_log = str_to_bool(options[:nolog]) + @options = options @logger = NoirLogger.new @is_debug, @is_color, @is_log end @@ -38,5 +38,50 @@ class Analyzer {% end %} end - define_getter_methods [result, base_path, url, scope, logger] + define_getter_methods [result, base_path, url, logger] +end + +class FileAnalyzer < Analyzer + @@hooks = [] of Proc(String, String, Array(Endpoint)) + + def hooks_count + @@hooks.size + end + + def self.add_hook(func : Proc(String, String, Array(Endpoint))) + @@hooks << func + end + + def analyze + channel = Channel(String).new + spawn do + Dir.glob("#{base_path}/**/*") do |file| + channel.send(file) + end + end + + @options[:concurrency].to_i.times do + spawn do + loop do + begin + path = channel.receive + next if File.directory?(path) + @@hooks.each do |hook| + file_results = hook.call(path, @url) + if !file_results.nil? + file_results.each do |file_result| + @result << file_result + end + end + end + rescue e + logger.debug e + end + end + end + end + + Fiber.yield + @result + end end diff --git a/src/models/endpoint.cr b/src/models/endpoint.cr index 3a790938..f104b2e1 100644 --- a/src/models/endpoint.cr +++ b/src/models/endpoint.cr @@ -4,17 +4,31 @@ require "yaml" struct Endpoint include JSON::Serializable include YAML::Serializable - property url, method, params, protocol + property url, method, params, protocol, details def initialize(@url : String, @method : String) + @params = [] of Param + @details = Details.new + @protocol = "http" + end + + def initialize(@url : String, @method : String, @details : Details) @params = [] of Param @protocol = "http" end def initialize(@url : String, @method : String, @params : Array(Param)) + @details = Details.new + @protocol = "http" + end + + def initialize(@url : String, @method : String, @params : Array(Param), @details : Details) @protocol = "http" end + def set_details(@details : Details) + end + def set_protocol(protocol : String) @protocol = protocol end @@ -48,6 +62,38 @@ struct Param end end +struct Details + include JSON::Serializable + include YAML::Serializable + property code_paths : Array(PathInfo) = [] of PathInfo + + # + New details types to be added in the future.. + + def initialize + end + + def initialize(code_path : PathInfo) + @code_paths << code_path + end + + def add_path(code_path : PathInfo) + @code_paths << code_path + end +end + +struct PathInfo + include JSON::Serializable + include YAML::Serializable + property path, line + + def initialize(@path : String) + @line = nil + end + + def initialize(@path : String, @line : Int32 | Nil) + end +end + struct EndpointReference include JSON::Serializable property endpoint, metadata diff --git a/src/models/noir.cr b/src/models/noir.cr index 16c655c3..46d9e014 100644 --- a/src/models/noir.cr +++ b/src/models/noir.cr @@ -4,6 +4,7 @@ require "../deliver/*" require "../output_builder/*" require "./endpoint.cr" require "./logger.cr" +require "../utils/string_extension.cr" require "json" class NoirRunner @@ -11,7 +12,6 @@ class NoirRunner @techs : Array(String) @endpoints : Array(Endpoint) @logger : NoirLogger - @scope : String @send_proxy : String @send_req : String @send_es : String @@ -37,7 +37,6 @@ class NoirRunner @send_proxy = options[:send_proxy] @send_req = options[:send_req] @send_es = options[:send_es] - @scope = options[:scope] @is_debug = str_to_bool(options[:debug]) @is_color = str_to_bool(options[:color]) @is_log = str_to_bool(options[:nolog]) @@ -72,7 +71,8 @@ class NoirRunner def analyze @endpoints = analysis_endpoints options, @techs, @logger optimize_endpoints - deliver() + combine_url_and_endpoints + deliver end def optimize_endpoints @@ -103,6 +103,35 @@ class NoirRunner @endpoints = tmp end + def combine_url_and_endpoints + tmp = [] of Endpoint + target_url = @options[:url] + + if target_url != "" + @logger.system "Combining url and endpoints." + @endpoints.each do |endpoint| + tmp_endpoint = endpoint + if tmp_endpoint.url.includes? target_url + tmp_endpoint.url = tmp_endpoint.url.gsub(target_url, "") + end + + tmp_endpoint.url = tmp_endpoint.url.gsub_repeatedly("//", "/") + if tmp_endpoint.url != "" + if target_url[-1] == '/' && tmp_endpoint.url[0] == '/' + tmp_endpoint.url = tmp_endpoint.url[1..] + elsif target_url[-1] != '/' && tmp_endpoint.url[0] != '/' + tmp_endpoint.url = "/#{tmp_endpoint.url}" + end + end + + tmp_endpoint.url = target_url + tmp_endpoint.url + tmp << tmp_endpoint + end + + @endpoints = tmp + end + end + def deliver if @send_proxy != "" @logger.system "Sending requests with proxy #{@send_proxy}." diff --git a/src/models/output_builder.cr b/src/models/output_builder.cr index 7c92500d..fd9600c8 100644 --- a/src/models/output_builder.cr +++ b/src/models/output_builder.cr @@ -6,7 +6,6 @@ class OutputBuilder @is_debug : Bool @is_color : Bool @is_log : Bool - @scope : String @output_file : String def initialize(options : Hash(Symbol, String)) @@ -14,7 +13,6 @@ class OutputBuilder @options = options @is_color = str_to_bool(options[:color]) @is_log = str_to_bool(options[:nolog]) - @scope = options[:scope] @output_file = options[:output] @logger = NoirLogger.new @is_debug, @is_color, @is_log @@ -34,9 +32,7 @@ class OutputBuilder end def bake_endpoint(url : String, params : Array(Param)) - if @is_debug - @logger.debug "Baking endpoint #{url} with #{params.size} params." - end + @logger.debug "Baking endpoint #{url} with #{params.size} params." final_url = url final_body = "" @@ -46,7 +42,7 @@ class OutputBuilder first_query = true first_form = true - if !params.nil? && @scope.includes?("param") + if !params.nil? params.each do |param| if param.param_type == "query" if first_query @@ -92,9 +88,7 @@ class OutputBuilder end end - if @is_debug - @logger.debug "Baked endpoint #{final_url} with #{final_body} body and #{final_headers.size} headers." - end + @logger.debug "Baked endpoint #{final_url} with #{final_body} body and #{final_headers.size} headers." { url: final_url, @@ -113,5 +107,5 @@ class OutputBuilder {% end %} end - define_getter_methods [scope, logger] + define_getter_methods [logger, output_file] end diff --git a/src/noir.cr b/src/noir.cr index 2cb6bac5..83b32965 100644 --- a/src/noir.cr +++ b/src/noir.cr @@ -6,7 +6,7 @@ require "./options.cr" require "./techs/techs.cr" module Noir - VERSION = "0.11.0" + VERSION = "0.12.0" end noir_options = default_options() @@ -17,13 +17,15 @@ OptionParser.parse do |parser| parser.separator " Basic:".colorize(:blue) parser.on "-b PATH", "--base-path ./app", "(Required) Set base path" { |var| noir_options[:base] = var } parser.on "-u URL", "--url http://..", "Set base url for endpoints" { |var| noir_options[:url] = var } - parser.on "-s SCOPE", "--scope url,param", "Set scope for detection" { |var| noir_options[:scope] = var } parser.separator "\n Output:".colorize(:blue) parser.on "-f FORMAT", "--format json", "Set output format \n[plain/json/yaml/markdown-table/curl/httpie/oas2/oas3]" { |var| noir_options[:format] = var } parser.on "-o PATH", "--output out.txt", "Write result to file" { |var| noir_options[:output] = var } parser.on "--set-pvalue VALUE", "Specifies the value of the identified parameter" { |var| noir_options[:set_pvalue] = var } + parser.on "--include-path", "Include file path in the plain result" do + noir_options[:include_path] = "yes" + end parser.on "--no-color", "Disable color output" do noir_options[:color] = "no" end @@ -102,32 +104,30 @@ noir_options.each do |k, v| end app.logger.debug "Initialized Options:" -app.logger.debug_sub "Base: #{app.options[:base]}" -app.logger.debug_sub "Techs: #{app.options[:techs]}" -app.logger.debug_sub "Scope: #{app.options[:scope]}" -app.logger.debug_sub "Send Proxy: #{app.@send_proxy}" -app.logger.debug_sub "Send Req: #{app.@send_req}" -app.logger.debug_sub "Debug: #{app.@is_debug}" -app.logger.debug_sub "Color: #{app.@is_color}" -app.logger.debug_sub "Format: #{app.options[:format]}" -app.logger.debug_sub "Output: #{app.options[:output]}" -app.logger.debug_sub "Concurrency: #{app.options[:concurrency]}" +app.options.each do |k, v| + app.logger.debug_sub "#{k}: #{v}" +end app.logger.system "Detecting technologies to base directory." app.detect + if app.techs.size == 0 app.logger.info "No technologies detected." - exit(1) + if app.options[:url] != "" + app.logger.system "Start file-based analysis as the -u flag has been used." + else + exit(0) + end else app.logger.info "Detected #{app.techs.size} technologies." app.techs.each do |tech| app.logger.info_sub "#{tech}" end + app.logger.system "Start code analysis based on the detected technology." +end - app.logger.system "Initiate code analysis based on the detected technology." - app.analyze - app.logger.info "Finally identified #{app.endpoints.size} endpoints." +app.analyze +app.logger.info "Finally identified #{app.endpoints.size} endpoints." - app.logger.system "Generating Report." - app.report -end +app.logger.system "Generating Report." +app.report diff --git a/src/options.cr b/src/options.cr index bd6d6fef..ff6497f3 100644 --- a/src/options.cr +++ b/src/options.cr @@ -1,9 +1,9 @@ def default_options noir_options = { :base => "", :url => "", :format => "plain", - :output => "", :techs => "", :debug => "no", :color => "yes", :concurrency => "100", + :output => "", :techs => "", :debug => "no", :color => "yes", :concurrency => "100", :include_path => "no", :send_proxy => "", :send_req => "no", :send_with_headers => "", :send_es => "", :use_matchers => "", :use_filters => "", - :scope => "url,param", :set_pvalue => "", :nolog => "no", + :set_pvalue => "", :nolog => "no", :exclude_techs => "", } diff --git a/src/output_builder/common.cr b/src/output_builder/common.cr index 36abc30a..37c08c76 100644 --- a/src/output_builder/common.cr +++ b/src/output_builder/common.cr @@ -31,6 +31,19 @@ class OutputBuilderCommon < OutputBuilder r_buffer += "\n ○ body: #{r_body}" end + if @options[:include_path] == "yes" + details = endpoint.details + if details.code_paths && details.code_paths.size > 0 + details.code_paths.each do |code_path| + if code_path.line.nil? + r_buffer += "\n ○ file: #{code_path.path}" + else + r_buffer += "\n ○ file: #{code_path.path}##{code_path.line}" + end + end + end + end + ob_puts r_buffer end end diff --git a/src/output_builder/markdown_table.cr b/src/output_builder/markdown_table.cr index 2889c587..54646708 100644 --- a/src/output_builder/markdown_table.cr +++ b/src/output_builder/markdown_table.cr @@ -7,7 +7,7 @@ class OutputBuilderMarkdownTable < OutputBuilder ob_puts "| -------- | -------- | ------ |" endpoints.each do |endpoint| - if !endpoint.params.nil? && @scope.includes?("param") + if !endpoint.params.nil? params_text = "" endpoint.params.each do |param| params_text += "`#{param.name} (#{param.param_type})` " diff --git a/src/techs/techs.cr b/src/techs/techs.cr index de06ee23..5effa54a 100644 --- a/src/techs/techs.cr +++ b/src/techs/techs.cr @@ -20,6 +20,11 @@ module NoirTechs :framework => "Gin", :similar => ["gin", "go-gin", "go_gin"], }, + :go_fiber => { + :language => "Go", + :framework => "Fiber", + :similar => ["fiber", "go-fiber", "go_fiber"], + }, :java_jsp => { :language => "Java", :framework => "JSP", diff --git a/src/utils/string_extension.cr b/src/utils/string_extension.cr new file mode 100644 index 00000000..0da2e472 --- /dev/null +++ b/src/utils/string_extension.cr @@ -0,0 +1,11 @@ +class String + def gsub_repeatedly(pattern, replacement) + result = self + if pattern != "" + while result.includes?(pattern) + result = result.gsub(pattern, replacement) + end + end + result + end +end