Skip to content

Commit

Permalink
implemented experimental draft of a feature request for performing th…
Browse files Browse the repository at this point in the history
…e transitive closure on the set of symbols originating from a specified filter set of libraries; added a try_goblin method to RdrObject; added command line flags for computing the transitive symbols, etc.;
  • Loading branch information
m4b committed Sep 24, 2015
1 parent 7e0f0e8 commit 40511b9
Show file tree
Hide file tree
Showing 16 changed files with 296 additions and 26 deletions.
7 changes: 4 additions & 3 deletions _oasis
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ Description:`rdr` is an OCaml tool/library for doing cross-platform analysis of

Features:

* 64-bit Linux and Mach-o binary analysis
* 64-bit Linux, Mach-o, 32-bit PE binary analysis
* Searchable symbol-map of all the symbols on your system, including binary
offset, size, and exporting library
* Print imports and exports of binaries
Expand All @@ -37,7 +37,7 @@ Library "utils"
FindLibParent: rdr
FindLibName: utils
CompiledObject: best
BuildDepends: unix
BuildDepends: unix, str
Modules:
RdrUtils,
Binary,
Expand Down Expand Up @@ -144,7 +144,8 @@ Library "rdr"
Rdr,
RdrObject,
RdrGraph,
SymbolMap
SymbolMap,
TransitiveClosure
BuildDepends:
rdr.goblin,
rdr.utils,
Expand Down
8 changes: 7 additions & 1 deletion _tags
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# OASIS_START
# DO NOT EDIT (digest: bd5d661cf9c00dd9ffbf7c369cf41597)
# DO NOT EDIT (digest: 6f72290905f846d782e04758586f28ab)
# Ignore VCS directories, you can use the same kind of rule outside
# OASIS_START/STOP if you want to exclude directories that contains
# useless stuff for the build process
Expand All @@ -16,28 +16,34 @@ true: annot, bin_annot
"_darcs": not_hygienic
# Library utils
"lib/utils/utils.cmxs": use_utils
<lib/utils/*.ml{,i,y}>: pkg_str
<lib/utils/*.ml{,i,y}>: pkg_unix
# Library mach
"lib/mach/mach.cmxs": use_mach
<lib/mach/*.ml{,i,y}>: pkg_str
<lib/mach/*.ml{,i,y}>: pkg_unix
<lib/mach/*.ml{,i,y}>: use_utils
# Library elf
"lib/elf/elf.cmxs": use_elf
<lib/elf/*.ml{,i,y}>: pkg_str
<lib/elf/*.ml{,i,y}>: pkg_unix
<lib/elf/*.ml{,i,y}>: use_utils
# Library pe
"lib/pe/pe.cmxs": use_pe
<lib/pe/*.ml{,i,y}>: pkg_str
<lib/pe/*.ml{,i,y}>: pkg_unix
<lib/pe/*.ml{,i,y}>: use_utils
# Library goblin
"lib/goblin/goblin.cmxs": use_goblin
<lib/goblin/*.ml{,i,y}>: pkg_str
<lib/goblin/*.ml{,i,y}>: pkg_unix
<lib/goblin/*.ml{,i,y}>: use_elf
<lib/goblin/*.ml{,i,y}>: use_mach
<lib/goblin/*.ml{,i,y}>: use_pe
<lib/goblin/*.ml{,i,y}>: use_utils
# Library rdr
"lib/rdr.cmxs": use_rdr
<lib/*.ml{,i,y}>: pkg_str
<lib/*.ml{,i,y}>: pkg_unix
<lib/*.ml{,i,y}>: use_elf
<lib/*.ml{,i,y}>: use_goblin
Expand Down
4 changes: 2 additions & 2 deletions lib/META
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# OASIS_START
# DO NOT EDIT (digest: aa2626bcb6c68a46ca94ec8f72308dcf)
# DO NOT EDIT (digest: 127233ddbb59488f5dacd5adbca956b4)
version = "3.0.0"
description =
"Lightweight, cross platform binary parsing and analysis library"
Expand All @@ -13,7 +13,7 @@ package "utils" (
version = "3.0.0"
description =
"Lightweight, cross platform binary parsing and analysis library"
requires = "unix"
requires = "unix str"
archive(byte) = "utils.cma"
archive(byte, plugin) = "utils.cma"
archive(native) = "utils.cmxa"
Expand Down
1 change: 1 addition & 0 deletions lib/Rdr.ml
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ module Utils = RdrUtils
module Object = RdrObject
module Map = SymbolMap
module Graph = RdrGraph
module TransitiveClosure = TransitiveClosure
11 changes: 11 additions & 0 deletions lib/RdrObject.ml
Original file line number Diff line number Diff line change
Expand Up @@ -101,3 +101,14 @@ let get ?verbose:(verbose=false) filename =
Printf.printf "ignoring binary: %s\n" filename;
Unknown (filename, (Printf.sprintf "unknown magic number 0x%x" magic))
end

let try_goblin ~coverage filename =
match get filename with
| Elf binary ->
Goblin.Elf.to_goblin ~coverage ~use_tree:true filename binary
| Mach binary ->
Goblin.Mach.to_goblin ~coverage filename binary
| PE32 binary ->
Goblin.PE.to_goblin ~coverage filename binary
| Unknown (_,error) -> raise @@
Failure (Printf.sprintf "<Rdr.Object.try_goblin> %s failed with %s" filename error)
8 changes: 4 additions & 4 deletions lib/SymbolMap.ml
Original file line number Diff line number Diff line change
Expand Up @@ -147,14 +147,14 @@ let build ~verbose:verbose ~graph:graph ~libs:libstack =
let library = Stack.pop libstack in
let binary =
try
match RdrObject.get ~verbose:verbose library with
match RdrObject.get ~verbose:false library with
| RdrObject.Mach binary ->
Some (Mach.get binary |> Goblin.Mach.from library)
Some (Mach.get ~coverage:false binary |> Goblin.Mach.from library)
| RdrObject.Elf binary ->
let elf = Elf.get binary in
let elf = Elf.get ~coverage:false binary in
Some (Goblin.Elf.from ~use_tree:false library elf)
| RdrObject.PE32 binary ->
Some (PE.get binary |> Goblin.PE.from library)
Some (PE.get ~coverage:false binary |> Goblin.PE.from library)
| RdrObject.Unknown (lib,error) ->
None
with _ ->
Expand Down
199 changes: 199 additions & 0 deletions lib/TransitiveClosure.ml
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
open Goblin
open Goblin.Import
open RdrUtils.Printer

(* TODO:
* add stats
* add graphing ?
*)

(*
with coverage
real 0m0.150s
user 0m0.103s
sys 0m0.040s
no coverage
real 0m0.148s
user 0m0.099s
sys 0m0.044s
*)
let debug = false

type transitive_symbol = {
name: string;
providers: string list;
}

let pp_transitive_symbol ppf ts =
Format.fprintf ppf "@[%s => "
ts.name;
pp_seq ~brackets:true ppf pp_string ts.providers;
Format.close_box()

module S = Set.Make(struct
type t = string
let compare ts1 ts2 = compare ts1 ts2
end)

module M = Map.Make(String)

let pp_set ppf (set:S.t) =
S.iter (fun elem -> Format.fprintf ppf "@ "; pp_string ppf elem) set
(* S.iter (pp_transitive_symbol ppf) set *)

let add filter (map, set) (imports:Goblin.Import.t array) =
Array.fold_left (fun (set,map) import ->
if (List.mem import.lib filter) then
let name = import.name in
let set = S.add name set in
let map =
if (M.mem name map) then
let providers = M.find name map in
M.add name (import.lib::providers) map
else
map
in
set,map
else
set,map
) (set,map) imports

let add_s set filter (imports:Goblin.Import.t array) =
Array.fold_left (fun set import ->
if (List.mem import.lib filter) then
let name = import.name in
let set = S.add name set in
set
else
set
) set imports

let lib_fallback_paths =
if (RdrCommand.is_osx()) then
["/usr/local/lib/"; "/usr/lib/"; "/lib/";]
else if (RdrCommand.is_linux()) then
["/usr/local/lib/"; "/usr/lib/";]
else
(* ... windows ? *)
["/usr/lib/";]

let rec close_libraries set visited filter libs =
let ppf = Format.std_formatter in
if (debug) then Format.fprintf Format.std_formatter "@[<v 2>@ ";
let solution = Array.fold_left (fun (set,visited) lib ->
if (debug) then begin
ignore @@ read_line() end;
let goblin,visited =
if (Sys.file_exists lib) then
if (not @@ S.mem lib visited) then
let visited = S.add lib visited in
if (debug) then
begin
Format.fprintf ppf "@ NOT Visited %s@ NEW Visited Set:@[<v 2>@ " lib;
pp_set Format.std_formatter visited;
Format.fprintf ppf "@]"
end;
Some (RdrObject.try_goblin ~coverage:false lib),visited
else
begin
if (debug) then Format.fprintf ppf "@ Visited %s" lib;
None,visited
end
else
let name = Filename.basename lib in
let fallback_libs = List.map ( fun x -> x ^ name) lib_fallback_paths in
let fallbacks_visited =
List.fold_left
(fun acc lib -> (S.mem lib visited) || acc) false fallback_libs
in
if (debug) then
begin
Format.fprintf Format.std_formatter "@ NOT FOUND %s@ Searching:@ " name;
Format.open_vbox 2;
Format.print_space();
pp_slist Format.std_formatter fallback_libs;
Format.print_space();
Format.close_box()
end;
if (not fallbacks_visited) then
fallback_search visited fallback_libs
else
None,visited
in
match goblin with
| Some goblin ->
let set = add_s set filter goblin.imports in
if (debug) then begin
Format.fprintf Format.std_formatter "@ Scanning %s's libaries:@ " (Filename.basename lib);
Format.open_vbox 2;
Format.print_space();
pp_slist Format.std_formatter (Array.to_list goblin.libs); Format.print_space();
Format.close_box()
end;
if (goblin.Goblin.libs = [||]) then
set,visited
else
close_libraries set visited filter goblin.libs
| None ->
(* TODO: refactor so actually unfound libraries are warned against, this is a serious issue if so *)
(* Format.eprintf "@ <Rdr.TransitiveClosure.close_libraries> lib %s not found@." lib; flush stdout; *)
set,visited
) (set,visited) libs
in
if (debug) then Format.fprintf Format.std_formatter "@]";
solution
and fallback_search visited libs =
match libs with
| [] -> None,visited
| lib::libs ->
try
if (not @@ S.mem lib visited) then
Some (RdrObject.try_goblin ~coverage:false lib),(S.add lib visited)
else
None,visited
with _ ->
fallback_search visited libs

(* solution *)
type t = {
symbols: string list;
transitive_library_dependencies: string list;
filter: string list;
}

let pp ppf t =
Format.fprintf ppf "@[@[<v 2>Transitive Closure filter@ ";
pp_slist ppf t.filter;
Format.fprintf ppf "@]";
Format.fprintf ppf "@[<v 2>Symbols@ ";
RdrUtils.Printer.pp_slist ppf t.symbols;
Format.fprintf ppf "@]";
Format.fprintf ppf "@[<v 2>Transitive Library Dependencies@ ";
RdrUtils.Printer.pp_slist ppf t.transitive_library_dependencies;
Format.fprintf ppf "@]@]@."

let print t =
pp Format.std_formatter t

(* TODO: do not need to scan the goblin's libraries, just the set of libraries created from every import's exporting library *)
let compute filename filter =
let goblin = RdrObject.try_goblin ~coverage:false filename in
let set = Array.fold_left (fun acc import ->
if (List.mem import.lib filter) then
let name = import.name in
(* let providers = [import.lib] in *)
S.add name acc
(* S.add {name; providers} acc *)
else
acc
) S.empty goblin.imports
in
let symbols,transitive_library_dependencies =
close_libraries set S.empty filter goblin.libs
in
{symbols = S.elements symbols;
transitive_library_dependencies = S.elements transitive_library_dependencies;
filter;
}
5 changes: 4 additions & 1 deletion lib/elf/Elf.ml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ type t = {
raw_code: bytes; (* list *)
}

let get ?meta_only:(meta_only=false) binary =
let get ?coverage:(coverage=true) ?meta_only:(meta_only=false) binary =
let header = Header.get_elf_header64 binary in
let entry = Int64.of_int header.Header.e_entry in
let is_64 = Header.is_64bit header.Header.e_ident in
Expand Down Expand Up @@ -103,7 +103,10 @@ let get ?meta_only:(meta_only=false) binary =
in
if (debug) then Reloc.print_relocs64 relocations;
let byte_coverage =
if (coverage) then
ElfCoverage.compute_byte_coverage header program_headers section_headers size binary
else
ByteCoverage.null
in
(* TODO: fix *)
let raw_code = if (meta_only) then
Expand Down
10 changes: 10 additions & 0 deletions lib/goblin/Goblin.ml
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,9 @@ module Mach = struct
let code = mach.Mach.raw_code in
{name; install_name; islib; libs; nlibs;
exports; nexports; imports; nimports; code}

let to_goblin ~coverage filename binary =
Mach.get ~coverage binary |> from filename
end

module Elf = struct
Expand Down Expand Up @@ -271,6 +274,10 @@ module Elf = struct
{name;
install_name; islib; libs; nlibs; exports; nexports;
imports; nimports; code}

let to_goblin ~coverage ~use_tree filename binary =
Elf.get ~coverage binary |> from ~use_tree filename

end

module PE = struct
Expand Down Expand Up @@ -316,5 +323,8 @@ module PE = struct
nexports = pe.nexports;
code = Bytes.empty;
}

let to_goblin ~coverage filename binary =
PE.get ~coverage binary |> from filename
end

2 changes: 1 addition & 1 deletion lib/goblin/GoblinTree.ml
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ let resolve_import ~case_sensitive:sensitive (branches: branch list) libraries =
| library::[] -> library
| library::libraries ->
Printf.sprintf
"Multiple Libraries Resolved\nThis is extremely dangerous, have fun!\n%s"
"Multiple Libraries Resolved\n%s"
@@ Generics.list_to_string acc
end
| branch::branches ->
Expand Down
Loading

0 comments on commit 40511b9

Please sign in to comment.