Merge pull request #142 from hahwul/dev

Release v0.10.0

README.md

@@ -35,6 +35,7 @@
 | Java     | Spring          | ✅  | ✅     | X     | X      | X  |
 | Kotlin   | Spring          | ✅  | ✅     | X     | X      | X  |
 | JS       | Express         | ✅  | ✅     | X     | X      | X  |
+| Rust     | Axum            | ✅  | ✅     | X     | X      | X  |
 | C#       | ASP.NET MVC     | ✅  | X      | X     | X      | X  |
 | JS       | Next            | X   | X      | X     | X      | X  |
 
@@ -101,6 +102,8 @@ Usage: noir <flags>
     --send-proxy http://proxy..      Send the results to the web request via http proxy
     --send-es https://es..           Send the results to elasticsearch
     --with-headers X-Header:Value    Add Custom Headers to be Used in Deliver
+    --use-matchers string            Delivers URLs that match a specific condition
+    --use-filters string             Excludes URLs that match a specific condition
 
   Technologies:
     -t TECHS, --techs rails,php      Set technologies to use

shard.yml

@@ -1,5 +1,5 @@
 name: noir
-version: 0.9.1
+version: 0.10.0
 
 authors:
   - hahwul <[email protected]>

spec/functional_test/fixtures/rust_axum/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "example-hello-world"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+axum = { path = "../../axum" }
+tokio = { version = "1.0", features = ["full"] }

spec/functional_test/fixtures/rust_axum/src/main.rs

@@ -0,0 +1,19 @@
+use axum::{response::Html, routing::get, Router};
+
+#[tokio::main]
+async fn main() {
+    let app = Router::new()
+        .route("/", get(handler))
+        .route("/foo", get(handler))
+        .route("/bar", post(handler));
+
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:3000")
+        .await
+        .unwrap();
+    println!("listening on {}", listener.local_addr().unwrap());
+    axum::serve(listener, app).await.unwrap();
+}
+
+async fn handler() -> Html<&'static str> {
+    Html("<h1>Hello, World!</h1>")
+}

spec/functional_test/testers/rust_axum_spec.cr

@@ -0,0 +1,12 @@
+require "../func_spec.cr"
+
+extected_endpoints = [
+  Endpoint.new("/", "GET"),
+  Endpoint.new("/foo", "GET"),
+  Endpoint.new("/bar", "POST"),
+]
+
+FunctionalTester.new("fixtures/rust_axum/", {
+  :techs     => 1,
+  :endpoints => 3,
+}, extected_endpoints).test_all

spec/unit_test/models/deliver_spec.cr

@@ -5,6 +5,7 @@ describe "Initialize" do
   options = default_options
   options[:base] = "noir"
   options[:send_proxy] = "http://localhost:8090"
+  options[:nolog] = "yes"
 
   it "Deliver" do
     object = Deliver.new options
@@ -16,4 +17,16 @@ describe "Initialize" do
     object = Deliver.new options
     object.headers["X-API-Key"].should eq("abcdssss")
   end
+
+  it "Deliver with matchers" do
+    options[:use_matchers] = "/admin"
+    object = Deliver.new options
+    object.matchers.should eq(["/admin"])
+  end
+
+  it "Deliver with filters" do
+    options[:use_filters] = "/admin"
+    object = Deliver.new options
+    object.filters.should eq(["/admin"])
+  end
 end

spec/utils_spec.cr

@@ -9,6 +9,20 @@ describe "remove_start_slash" do
   end
 end
 
+describe "get_relative_path" do
+  it "start with ./" do
+    get_relative_path("./abcd", "1.cr").should eq("1.cr")
+  end
+
+  it "start with /" do
+    get_relative_path("/abcd", "1.cr").should eq("1.cr")
+  end
+
+  it "end with /" do
+    get_relative_path("/abcd/", "1.cr").should eq("1.cr")
+  end
+end
+
 describe "get_symbol" do
   it "get" do
     get_symbol("GET").should eq(:get)

src/analyzer/analyzer.cr

@@ -20,6 +20,7 @@ def initialize_analyzers(logger : NoirLogger)
   analyzers["raml"] = ->analyzer_raml(Hash(Symbol, String))
   analyzers["java_jsp"] = ->analyzer_jsp(Hash(Symbol, String))
   analyzers["c#-aspnet-mvc"] = ->analyzer_cs_aspnet_mvc(Hash(Symbol, String))
+  analyzers["rust_axum"] = ->analyzer_rust_axum(Hash(Symbol, String))
 
   logger.info_sub "#{analyzers.size} Analyzers initialized"
   logger.debug "Analyzers:"

src/analyzer/analyzers/analyzer_jsp.cr

@@ -7,12 +7,8 @@ class AnalyzerJsp < Analyzer
     begin
       Dir.glob("#{base_path}/**/*") do |path|
         next if File.directory?(path)
-        if base_path[-1].to_s == "/"
-          relative_path = path.sub("#{base_path}", "").sub("./", "").sub("//", "/")
-        else
-          relative_path = path.sub("#{base_path}/", "").sub("./", "").sub("//", "/")
-        end
-        relative_path = remove_start_slash(relative_path)
+
+        relative_path = get_relative_path(base_path, path)
 
         if File.exists?(path) && File.extname(path) == ".jsp"
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|

src/analyzer/analyzers/analyzer_php_pure.cr

@@ -7,12 +7,8 @@ class AnalyzerPhpPure < Analyzer
     begin
       Dir.glob("#{base_path}/**/*") do |path|
         next if File.directory?(path)
-        if base_path[-1].to_s == "/"
-          relative_path = path.sub("#{base_path}", "").sub("./", "").sub("//", "/")
-        else
-          relative_path = path.sub("#{base_path}/", "").sub("./", "").sub("//", "/")
-        end
-        relative_path = remove_start_slash(relative_path)
+
+        relative_path = get_relative_path(base_path, path)
 
         if File.exists?(path) && File.extname(path) == ".php"
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|

src/analyzer/analyzers/analyzer_rust_axum.cr

@@ -0,0 +1,49 @@
+require "../../models/analyzer"
+
+class AnalyzerRustAxum < Analyzer
+  def analyze
+    # Source Analysis
+    pattern = /\.route\("([^"]+)",\s*([^)]+)\)/
+
+    begin
+      Dir.glob("#{base_path}/**/*") do |path|
+        next if File.directory?(path)
+
+        if File.exists?(path) && File.extname(path) == ".rs"
+          File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
+            file.each_line do |line|
+              if line.includes? ".route("
+                match = line.match(pattern)
+                if match
+                  begin
+                    route_argument = match[1]
+                    callback_argument = match[2]
+                    result << Endpoint.new("#{@url}#{route_argument}", callback_to_method(callback_argument))
+                  rescue
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    rescue e
+    end
+
+    result
+  end
+
+  def callback_to_method(str)
+    method = str.split("(").first
+    if !["get", "post", "put", "delete"].includes?(method)
+      method = "get"
+    end
+
+    method.upcase
+  end
+end
+
+def analyzer_rust_axum(options : Hash(Symbol, String))
+  instance = AnalyzerRustAxum.new(options)
+  instance.analyze
+end

src/deliver/send_elasticsearch.cr

@@ -9,8 +9,10 @@ class SendElasticSearch < Deliver
       uri.port = 9200
     end
 
+    applied_endpoints = apply_all(endpoints)
+
     body = {
-      "endpoints" => endpoints,
+      "endpoints" => applied_endpoints,
     }.to_json
     es_headers = @headers
     es_headers["Content-Type"] = "application/json"

src/deliver/send_proxy.cr

@@ -5,7 +5,8 @@ require "../models/deliver"
 class SendWithProxy < Deliver
   def run(endpoints : Array(Endpoint))
     proxy_url = URI.parse(@proxy)
-    endpoints.each do |endpoint|
+    applied_endpoints = apply_all(endpoints)
+    applied_endpoints.each do |endpoint|
       begin
         if endpoint.params.size > 0
           endpoint_hash = endpoint.params_to_hash

src/deliver/send_req.cr

@@ -4,7 +4,8 @@ require "../models/deliver"
 
 class SendReq < Deliver
   def run(endpoints : Array(Endpoint))
-    endpoints.each do |endpoint|
+    applied_endpoints = apply_all(endpoints)
+    applied_endpoints.each do |endpoint|
       begin
         if endpoint.params.size > 0
           endpoint_hash = endpoint.params_to_hash

src/detector/detector.cr

@@ -17,6 +17,7 @@ def detect_techs(base_path : String, options : Hash(Symbol, String), logger : No
     DetectorPhpPure, DetectorPythonDjango, DetectorPythonFlask, DetectorPythonFastAPI,
     DetectorRubyRails, DetectorRubySinatra, DetectorOas2, DetectorOas3, DetectorRAML,
     DetectorGoGin, DetectorKotlinSpring, DetectorJavaArmeria, DetectorCSharpAspNetMvc,
+    DetectorRustAxum,
   ])
   begin
     Dir.glob("#{base_path}/**/*") do |file|

src/detector/detectors/rust_axum.cr

@@ -0,0 +1,15 @@
+require "../../models/detector"
+
+class DetectorRustAxum < Detector
+  def detect(filename : String, file_contents : String) : Bool
+    check = file_contents.includes?("axum")
+    check = check && file_contents.includes?("dependencies")
+    check = check && filename.includes?("Cargo.toml")
+
+    check
+  end
+
+  def set_name
+    @name = "rust_axum"
+  end
+end

src/models/deliver.cr

@@ -8,6 +8,8 @@ class Deliver
   @is_log : Bool
   @proxy : String
   @headers : Hash(String, String) = {} of String => String
+  @matchers : Array(String) = [] of String
+  @filters : Array(String) = [] of String
 
   def initialize(options : Hash(Symbol, String))
     @options = options
@@ -29,6 +31,65 @@ class Deliver
       end
       @logger.info_sub "#{@headers.size} headers added."
     end
+
+    @matchers = options[:use_matchers].split("::NOIR::MATCHER::SPLIT::")
+    @matchers.delete("")
+    if @matchers.size > 0
+      @logger.system "#{@matchers.size} matchers added."
+    end
+
+    @filters = options[:use_filters].split("::NOIR::FILTER::SPLIT::")
+    @filters.delete("")
+    if @filters.size > 0
+      @logger.system "#{@filters.size} filters added."
+    end
+  end
+
+  def apply_all(endpoints : Array(Endpoint))
+    result = endpoints
+    @logger.debug "Matchers: #{@matchers}"
+    @logger.debug "Filters: #{@filters}"
+
+    if @matchers.size > 0
+      @logger.system "Applying matchers"
+      result = apply_matchers(endpoints)
+    end
+
+    if @filters.size > 0
+      @logger.system "Applying filters"
+      result = apply_filters(endpoints)
+    end
+
+    result
+  end
+
+  def apply_matchers(endpoints : Array(Endpoint))
+    result = [] of Endpoint
+    endpoints.each do |endpoint|
+      @matchers.each do |matcher|
+        if endpoint.url.includes? matcher
+          @logger.debug "Endpoint '#{endpoint.url}' matched with '#{matcher}'."
+          result << endpoint
+        end
+      end
+    end
+
+    result
+  end
+
+  def apply_filters(endpoints : Array(Endpoint))
+    result = [] of Endpoint
+    endpoints.each do |endpoint|
+      @filters.each do |filter|
+        if endpoint.url.includes? filter
+          @logger.debug "Endpoint '#{endpoint.url}' filtered with '#{filter}'."
+        else
+          result << endpoint
+        end
+      end
+    end
+
+    result
   end
 
   def proxy
@@ -39,6 +100,14 @@ class Deliver
     @headers
   end
 
+  def matchers
+    @matchers
+  end
+
+  def filters
+    @filters
+  end
+
   def run
     # After inheriting the class, write an action code here.
   end

src/noir.cr

@@ -6,7 +6,7 @@ require "./options.cr"
 require "./techs/techs.cr"
 
 module Noir
-  VERSION = "0.9.1"
+  VERSION = "0.10.0"
 end
 
 noir_options = default_options()
@@ -38,6 +38,12 @@ OptionParser.parse do |parser|
   parser.on "--with-headers X-Header:Value", "Add Custom Headers to be Used in Deliver" do |var|
     noir_options[:send_with_headers] += "#{var}::NOIR::HEADERS::SPLIT::"
   end
+  parser.on "--use-matchers string", "Delivers URLs that match a specific condition" do |var|
+    noir_options[:use_matchers] += "#{var}::NOIR::MATCHER::SPLIT::"
+  end
+  parser.on "--use-filters string", "Excludes URLs that match a specific condition" do |var|
+    noir_options[:use_filters] += "#{var}::NOIR::FILTER::SPLIT::"
+  end
 
   parser.separator "\n  Technologies:".colorize(:blue)
   parser.on "-t TECHS", "--techs rails,php", "Specify the technologies to use" { |var| noir_options[:techs] = var }

src/options.cr

@@ -2,7 +2,7 @@ def default_options
   noir_options = {
     :base => "", :url => "", :format => "plain",
     :output => "", :techs => "", :debug => "no", :color => "yes",
-    :send_proxy => "", :send_req => "no", :send_with_headers => "", :send_es => "",
+    :send_proxy => "", :send_req => "no", :send_with_headers => "", :send_es => "", :use_matchers => "", :use_filters => "",
     :scope => "url,param", :set_pvalue => "", :nolog => "no",
     :exclude_techs => "",
   }

src/techs/techs.cr

@@ -75,6 +75,11 @@ module NoirTechs
       :framework => "Sinatra",
       :similar   => ["sinatra", "ruby-sinatra", "ruby_sinatra"],
     },
+    :rust_axum => {
+      :language  => "Rust",
+      :framework => "Axum",
+      :similar   => ["axum", "rust-axum", "rust_axum"],
+    },
     :oas2 => {
       :format  => ["JSON", "YAML"],
       :similar => ["oas 2.0", "oas_2_0", "swagger 2.0", "swagger_2_0", "swagger"],