From 86fdb91b7c6f8b7aeccb7704d8b9209bf63d4acf Mon Sep 17 00:00:00 2001 From: Thomas Gazagnaire Date: Fri, 19 May 2017 16:38:54 -0700 Subject: [PATCH 1/5] GitHub bridge: allow git:// urls to "connect" to a local Git repo This is useful to run the bridge with one command and does not require to have a DataKit server running. Signed-off-by: Thomas Gazagnaire --- bridge/github/jbuild | 1 + bridge/github/main.ml | 82 +++++++++++++++++++++++++------------- datakit-bridge-github.opam | 1 + 3 files changed, 56 insertions(+), 28 deletions(-) diff --git a/bridge/github/jbuild b/bridge/github/jbuild index 745a4add1..0a6f2b9fa 100644 --- a/bridge/github/jbuild +++ b/bridge/github/jbuild @@ -13,6 +13,7 @@ (package datakit-bridge-github) (public_name datakit-bridge-github) (libraries (datakit_bridge_github datakit-client-9p protocol-9p-unix + datakit-client-git prometheus-app.unix datakit_log fmt.cli fmt.tty github.unix)))) ; TODO: generate the right version using topkg watermarking diff --git a/bridge/github/main.ml b/bridge/github/main.ml index c21aa2dfc..578828a7d 100644 --- a/bridge/github/main.ml +++ b/bridge/github/main.ml @@ -41,11 +41,14 @@ let quiet () = module Client9p = Protocol_9p_unix.Client9p_unix.Make(Log9p) module DK = Datakit_client_9p.Make(Client9p) module Sync = Datakit_github_sync.Make(Datakit_github_api)(DK) +module Sync_git = + Datakit_github_sync.Make(Datakit_github_api)(Datakit_client_git) let token () = let cookie = "datakit-github-cookie" in let error_msg = - Fmt.strf "Missing cookie %s/%s: use `git-jar make -s repo %s`" jar_path cookie cookie in + Fmt.strf "Missing cookie %s/%s: use `git-jar make -s repo %s`" + jar_path cookie cookie in Lwt_main.run ( Lwt.catch (fun () -> let open Lwt.Infix in @@ -65,11 +68,6 @@ let set_signal_if_supported signal handler = with Invalid_argument _ -> () -let ( >>~ ) x f = - x >>= function - | Error e -> Lwt.fail_with @@ Fmt.strf "%a" DK.pp_error e - | Ok t -> f t - let () = Lwt.async_exception_hook := (fun exn -> Logs.err (fun m -> m "Unhandled exception: %a" Fmt.exn exn) @@ -78,21 +76,24 @@ let () = type endpoint = [ | `File of string | `Tcp of string * int + | `Git of string ] let pp_endpoint ppf = function | `File f -> Fmt.pf ppf "file://%s" f | `Tcp (h, p) -> Fmt.pf ppf "tcp://%s:%d" h p + | `Git d -> Fmt.pf ppf "git://%s" d let endpoint_of_string ~default_tcp_port str = match String.cut ~sep:"://" str with | Some ("file", f) -> `Ok (`File f) | Some ("tcp" , s) -> (match String.cut ~rev:true ~sep:":" s with - | None -> `Ok (`Tcp (s, default_tcp_port)) - | Some (host, port) -> - try `Ok (`Tcp (host, int_of_string port)) - with Failure _ -> `Error "use tcp://host:port") + | None -> `Ok (`Tcp (s, default_tcp_port)) + | Some (host, port) -> + try `Ok (`Tcp (host, int_of_string port)) + with Failure _ -> `Error "use tcp://host:port") + | Some ("git", d) -> `Ok (`Git d) | _ -> `Error "invalid endpoint" type datakit_config = { @@ -100,6 +101,48 @@ type datakit_config = { branch : string; } +let connect_9p ~token ?webhook ?resync_interval ~cap ~branch endpoint = + let proto, address = match endpoint with + | `Tcp (host, port) -> "tcp" , Fmt.strf "%s:%d" host port + | `File path -> "unix", path (* FIXME: weird proto name for 9p *) + in + Lwt.catch + (fun () -> + Client9p.connect + proto address ~send_pings:true ~max_fids:Int32.max_int ()) + (fun e -> Lwt.fail_with @@ Fmt.strf "%a" Fmt.exn e) + >>= function + | Error (`Msg e) -> + Log.err (fun l -> l "cannot connect: %s" e); + Lwt.fail_with "connecting to datakit" + | Ok conn -> + Log.info (fun l -> l "Connected to %a" pp_endpoint endpoint); + let dk = DK.connect conn in + let t = Sync.empty in + DK.branch dk branch >>= function + | Error e -> Lwt.fail_with @@ Fmt.strf "%a" DK.pp_error e + | Ok br -> + Sync.sync ~token ?webhook ?resync_interval ~cap br t + >|= ignore + +let connect_git ~token ?webhook ?resync_interval ~cap ~branch root = + Datakit_client_git.connect ~head:("refs/heads/" ^ branch) ~bare:false root + >>= fun dk -> + let t = Sync_git.empty in + Datakit_client_git.branch dk branch >>= function + | Error e -> Lwt.fail_with @@ Fmt.strf "%a" Datakit_client_git.pp_error e + | Ok br -> + Sync_git.sync ~token ?webhook ?resync_interval ~cap br t + >|= ignore + +let connect ~token ?webhook ?resync_interval ~cap d = + let branch = d.branch in + match d.endpoint with + | `Tcp _ | `File _ as x -> + connect_9p ~token ?webhook ?resync_interval ~cap ~branch x + | `Git root -> + connect_git ~token ?webhook ?resync_interval ~cap ~branch root + let start () datakit cap webhook resync_interval prometheus = quiet (); let prometheus_threads = Prometheus_unix.serve prometheus in @@ -137,24 +180,7 @@ let start () datakit cap webhook resync_interval prometheus = | Some d -> Log.app (fun l -> l "Connecting to %a [%s]." pp_endpoint d.endpoint d.branch); - let proto, address = match d.endpoint with - | `Tcp (host, port) -> "tcp" , Fmt.strf "%s:%d" host port - | `File path -> "unix", path (* FIXME: weird proto name for 9p *) - in - Lwt.catch - (fun () -> Client9p.connect proto address ~send_pings:true ~max_fids:Int32.max_int ()) - (fun e -> Lwt.fail_with @@ Fmt.strf "%a" Fmt.exn e) - >>= function - | Error (`Msg e) -> - Log.err (fun l -> l "cannot connect: %s" e); - Lwt.fail_with "connecting to datakit" - | Ok conn -> - Log.info (fun l -> l "Connected to %a" pp_endpoint d.endpoint); - let dk = DK.connect conn in - let t = Sync.empty in - DK.branch dk d.branch >>~ fun br -> - Sync.sync ~token ?webhook ?resync_interval ~cap br t - >|= ignore + connect ~token ?webhook ?resync_interval ~cap d in Lwt_main.run @@ Lwt.choose ( [connect_to_datakit ()] @ prometheus_threads diff --git a/datakit-bridge-github.opam b/datakit-bridge-github.opam index bdab9585a..7e39ae499 100644 --- a/datakit-bridge-github.opam +++ b/datakit-bridge-github.opam @@ -17,6 +17,7 @@ depends: [ "lwt" {>= "3.0.0"} "datakit-github" {>= "0.11.0"} "datakit-client-9p" {>= "0.11.0"} + "datakit-client-git" "logs" "fmt" "mtime" {>= "1.0.0"} "asl" "win-eventlog" From cd578cedc3d96cd1577113b4841c37b1afc7799c Mon Sep 17 00:00:00 2001 From: Thomas Gazagnaire Date: Fri, 19 May 2017 18:31:10 -0700 Subject: [PATCH 2/5] GitHub bridge: look at the GH token in various places The current default is sensible when running in Docker: it was not when run as a normal binary. Signed-off-by: Thomas Gazagnaire --- bridge/github/main.ml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bridge/github/main.ml b/bridge/github/main.ml index 578828a7d..27bf860b0 100644 --- a/bridge/github/main.ml +++ b/bridge/github/main.ml @@ -8,7 +8,14 @@ module Log = (val Logs.src_log src : Logs.LOG) let src9p = Logs.Src.create "g9p" ~doc:"Github bridge for Datakit (9p)" module Log9p = (val Logs.src_log src9p : Logs.LOG) -let jar_path = "/run/secrets" +let jar_paths = [ + (try Sys.getenv "HOME" with Not_found -> "") ^ "/.github/jar"; + "/run/secrets"; +] + +let jar_path = + try List.find Sys.file_exists jar_paths + with Not_found -> List.hd jar_paths let quiet_9p () = Logs.Src.set_level src9p (Some Logs.Info); From c2ecf55021b220290084ca30f62c3a22d7263f58 Mon Sep 17 00:00:00 2001 From: Thomas Gazagnaire Date: Fri, 19 May 2017 18:33:16 -0700 Subject: [PATCH 3/5] GitHub bridge: add the ability to monitor a repo on the CLI Signed-off-by: Thomas Gazagnaire --- bridge/github/main.ml | 47 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/bridge/github/main.ml b/bridge/github/main.ml index 27bf860b0..e15a65ee7 100644 --- a/bridge/github/main.ml +++ b/bridge/github/main.ml @@ -108,7 +108,26 @@ type datakit_config = { branch : string; } -let connect_9p ~token ?webhook ?resync_interval ~cap ~branch endpoint = +let monitor (type a) (module DK: Datakit_client.S with type Branch.t = a) + (br: a) repos = + let module Conv = Datakit_github_conv.Make(DK) in + if repos <> [] then ( + DK.Branch.with_transaction br (fun tr -> + Lwt_list.iter_p (fun r -> + match Datakit_github.Repo.of_string r with + | None -> Lwt.return_unit + | Some r -> Conv.update_elt tr (`Repo r) + ) repos + >>= fun () -> + DK.Transaction.commit tr ~message:"initial commit" + ) >>= function + | Error e -> Fmt.kstrf Lwt.fail_with "%a" DK.pp_error e + | Ok () -> Lwt.return_unit + ) else + Lwt.return_unit + +let connect_9p ~token ?webhook ?resync_interval ~cap ~branch ~repositories + endpoint = let proto, address = match endpoint with | `Tcp (host, port) -> "tcp" , Fmt.strf "%s:%d" host port | `File path -> "unix", path (* FIXME: weird proto name for 9p *) @@ -129,28 +148,31 @@ let connect_9p ~token ?webhook ?resync_interval ~cap ~branch endpoint = DK.branch dk branch >>= function | Error e -> Lwt.fail_with @@ Fmt.strf "%a" DK.pp_error e | Ok br -> + monitor (module DK) br repositories >>= fun () -> Sync.sync ~token ?webhook ?resync_interval ~cap br t >|= ignore -let connect_git ~token ?webhook ?resync_interval ~cap ~branch root = +let connect_git ~token ?webhook ?resync_interval ~cap ~branch ~repositories + root = Datakit_client_git.connect ~head:("refs/heads/" ^ branch) ~bare:false root >>= fun dk -> let t = Sync_git.empty in Datakit_client_git.branch dk branch >>= function | Error e -> Lwt.fail_with @@ Fmt.strf "%a" Datakit_client_git.pp_error e | Ok br -> + monitor (module Datakit_client_git) br repositories >>= fun () -> Sync_git.sync ~token ?webhook ?resync_interval ~cap br t >|= ignore -let connect ~token ?webhook ?resync_interval ~cap d = +let connect ~token ?webhook ?resync_interval ~repositories ~cap d = let branch = d.branch in match d.endpoint with | `Tcp _ | `File _ as x -> - connect_9p ~token ?webhook ?resync_interval ~cap ~branch x + connect_9p ~token ?webhook ?resync_interval ~cap ~branch ~repositories x | `Git root -> - connect_git ~token ?webhook ?resync_interval ~cap ~branch root + connect_git ~token ?webhook ?resync_interval ~cap ~branch ~repositories root -let start () datakit cap webhook resync_interval prometheus = +let start () datakit repositories cap webhook resync_interval prometheus = quiet (); let prometheus_threads = Prometheus_unix.serve prometheus in set_signal_if_supported Sys.sigpipe Sys.Signal_ignore; @@ -187,7 +209,7 @@ let start () datakit cap webhook resync_interval prometheus = | Some d -> Log.app (fun l -> l "Connecting to %a [%s]." pp_endpoint d.endpoint d.branch); - connect ~token ?webhook ?resync_interval ~cap d + connect ~token ?webhook ?resync_interval ~cap ~repositories d in Lwt_main.run @@ Lwt.choose ( [connect_to_datakit ()] @ prometheus_threads @@ -282,6 +304,14 @@ let capabilities = in Arg.(value & opt cap Datakit_github.Capabilities.all doc) +let repositories = + let docs = github_options in + let doc = + Arg.info ~docs ~doc:"A list of repository to monitor on startup." + ["repositories";"r"] + in + Arg.(value & opt (list string) [] doc) + let term = let doc = "Bridge between GitHub API and Datakit." in let man = [ @@ -291,7 +321,8 @@ let term = bidirectional mapping between the GitHub API and a Git branch."; ] in Term.(pure start $ setup_log - $ datakit $ capabilities $ webhook $ resync $ Prometheus_unix.opts), + $ datakit $ repositories + $ capabilities $ webhook $ resync $ Prometheus_unix.opts), Term.info (Filename.basename Sys.argv.(0)) ~version:Version.v ~doc ~man let () = match Term.eval term with From 948732c3cf52807899e9806e18284c6420a41141 Mon Sep 17 00:00:00 2001 From: Thomas Gazagnaire Date: Tue, 30 May 2017 08:55:31 +0200 Subject: [PATCH 4/5] github-bridge: fix Dockerfile Signed-off-by: Thomas Gazagnaire --- Dockerfile.github | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile.github b/Dockerfile.github index 88d42d3d4..2c7c1c924 100644 --- a/Dockerfile.github +++ b/Dockerfile.github @@ -7,6 +7,7 @@ ENV OPAMERRLOGLEN=0 OPAMYES=1 RUN sudo apk add tzdata aspcud RUN opam pin add git.dev --dev -n +RUN opam pin add git-unix.dev -n --dev RUN opam pin add irmin.1.2.0 -n https://github.com/mirage/irmin.git#81ffa256d3aa6b73b689609f7a5ff01c298fe821 RUN opam pin add irmin-mem.1.2.0 -n https://github.com/mirage/irmin.git#81ffa256d3aa6b73b689609f7a5ff01c298fe821 @@ -23,6 +24,7 @@ RUN opam config exec -- ocaml /tmp/check-libev.ml COPY datakit.opam /home/opam/src/datakit/datakit.opam COPY datakit-client.opam /home/opam/src/datakit/datakit-client.opam COPY datakit-client-9p.opam /home/opam/src/datakit/datakit-client-9p.opam +COPY datakit-client-git.opam /home/opam/src/datakit/datakit-client-git.opam COPY datakit-server.opam /home/opam/src/datakit/datakit-server.opam COPY datakit-server-9p.opam /home/opam/src/datakit/datakit-server-9p.opam COPY datakit-github.opam /home/opam/src/datakit/datakit-github.opam @@ -32,6 +34,7 @@ RUN opam pin add datakit.dev /home/opam/src/datakit -yn && \ opam pin add datakit-server-9p.dev /home/opam/src/datakit -yn && \ opam pin add datakit-client.dev /home/opam/src/datakit -yn && \ opam pin add datakit-client-9p.dev /home/opam/src/datakit -yn && \ + opam pin add datakit-client-git.dev /home/opam/src/datakit -yn && \ opam pin add datakit-github.dev /home/opam/src/datakit -yn && \ opam pin add datakit-bridge-github.dev /home/opam/src/datakit -yn From c67ba3f680811f692ba1a7170f302010dc4ff4a4 Mon Sep 17 00:00:00 2001 From: Thomas Gazagnaire Date: Tue, 30 May 2017 09:00:11 +0200 Subject: [PATCH 5/5] Dockerfiles: use latest ocaml/opam-dev images to workaround a bug in opam2 update Signed-off-by: Thomas Gazagnaire --- Dockerfile | 5 +---- Dockerfile.ci | 7 ++----- Dockerfile.client | 5 +---- Dockerfile.github | 6 ++---- 4 files changed, 6 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 956489dc8..4b0e161b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,4 @@ -FROM ocaml/opam-dev@sha256:64dc0522876ebbd27b186e3ba955ae5ab864ace580add5b0d1abb8715bdf1bbe -RUN git -C /home/opam/opam-repository fetch origin && \ - git -C /home/opam/opam-repository reset origin/master --hard && \ - opam update -u +FROM ocaml/opam-dev@sha256:1dd4440b3e5f182f705cb5a74f9d4e860c2842b45ed72c199de89a894d13f522 #FROM ocaml/opam-dev:alpine-3.5_ocaml-4.04.0 ENV OPAMERRLOGLEN=0 OPAMYES=1 diff --git a/Dockerfile.ci b/Dockerfile.ci index 8fd457e2f..7ec80b27d 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,9 +1,6 @@ -FROM ocaml/opam-dev@sha256:64dc0522876ebbd27b186e3ba955ae5ab864ace580add5b0d1abb8715bdf1bbe -RUN git -C /home/opam/opam-repository fetch origin && \ - git -C /home/opam/opam-repository reset origin/master --hard && \ - git -C /home/opam/opam-repository rev-parse HEAD && \ - opam update -u +FROM ocaml/opam-dev@sha256:1dd4440b3e5f182f705cb5a74f9d4e860c2842b45ed72c199de89a894d13f522 #FROM ocaml/opam-dev:alpine-3.5_ocaml-4.04.0 + ENV OPAMERRLOGLEN=0 OPAMYES=1 RUN sudo apk add tzdata aspcud diff --git a/Dockerfile.client b/Dockerfile.client index bac65409a..9c17cd9f9 100644 --- a/Dockerfile.client +++ b/Dockerfile.client @@ -1,7 +1,4 @@ -FROM ocaml/opam-dev@sha256:64dc0522876ebbd27b186e3ba955ae5ab864ace580add5b0d1abb8715bdf1bbe -RUN git -C /home/opam/opam-repository fetch origin && \ - git -C /home/opam/opam-repository reset origin/master --hard && \ - opam update -u +FROM ocaml/opam-dev@sha256:1dd4440b3e5f182f705cb5a74f9d4e860c2842b45ed72c199de89a894d13f522 #FROM ocaml/opam:alpine-3.5_ocaml-4.04.0 ENV OPAMERRLOGLEN=0 OPAMYES=1 diff --git a/Dockerfile.github b/Dockerfile.github index 2c7c1c924..a97289f41 100644 --- a/Dockerfile.github +++ b/Dockerfile.github @@ -1,8 +1,6 @@ -FROM ocaml/opam-dev@sha256:64dc0522876ebbd27b186e3ba955ae5ab864ace580add5b0d1abb8715bdf1bbe -RUN git -C /home/opam/opam-repository fetch origin && \ - git -C /home/opam/opam-repository reset origin/master --hard && \ - opam update -u +FROM ocaml/opam-dev@sha256:1dd4440b3e5f182f705cb5a74f9d4e860c2842b45ed72c199de89a894d13f522 #FROM ocaml/opam-dev:alpine-3.5_ocaml-4.04.0 + ENV OPAMERRLOGLEN=0 OPAMYES=1 RUN sudo apk add tzdata aspcud