Add support to include repositories (#359)

This patch adds support to both add include and exclude expressions to
filter repositories. The repository names are first run through the
include and then exclude filters. By default all repositories are
included.

Fixes #171

Co-authored-by: Frank Schroeder <[email protected]>

README.md

@@ -78,7 +78,7 @@ nix shell nixpkgs#git-workspace
 
 ## Binaries (Windows)
 
-Download the latest release from [the github releases page](https://github.com/orf/git-workspace/releases). Extract it 
+Download the latest release from [the github releases page](https://github.com/orf/git-workspace/releases). Extract it
 and move it to a directory on your `PATH`.
 
 ## Cargo
@@ -147,19 +147,23 @@ You can use `git workspace add` to quickly add entries to your `workspace.toml`:
 
    * `git workspace add github [USER OR ORG NAME]`
 
-* Exclude specific repositories:
+* Include and exclude specific repositories:
 
-   * `git workspace add github [USER OR ORG NAME] --exclude="foo.*bar$" --exclude="(abc|def)"`
+   * `git workspace add github [USER OR ORG NAME] --include="a.*$" --include="b.*$" --exclude="aa.*$"  --exclude="bb.*$"`
 
-* Clone a namespace or user from Gitlab: 
+   * Both `--include` and `--exclude` can be specified multiple times.
+   * By default all repositories are included.
+   * All `include` filters are evaluated before the `exclude` filters.
+
+* Clone a namespace or user from Gitlab:
 
    * `git workspace add gitlab gitlab-ce/gitlab-services`
 
-* Clone from a self-hosted gitlab/github instance: 
+* Clone from a self-hosted gitlab/github instance:
 
    * `git workspace add gitlab my-company-group --url=https://internal-gitlab.company.com`
    * `git workspace add github user-or-org-name --url=https://internal-github.company.com/api/graphql`
-   
+
 ### Multiple configs
 
 Git workspace will read from any `workspace*.toml` file under your `$GIT_WORKSPACE` directory.
@@ -224,10 +228,10 @@ wsp() {
     for command in ${FZF:-"fzf"} ${GIT:-"git"}; do
         ${COMMAND:-"command"} -v "$command" || { ${PRINTF:-"printf"} "FATAL: %s\\n" "Command '$command' is not executable"; ${EXIT:-"exit"} 127 ;}
     done
-    
+
     # shellcheck disable=SC2086 # Harmless warning about missing double-quotes that are not expected to allow parsing multiple arguments
     wsp_path="${1:-"${GTT_WORKSPACE:-"$PWD"}/$(${GIT:-"git"} workspace list | ${FZF:-"fzf"} ${fzf_arg:-"-q"} "$@")"}" # Path to the git workspace directory
-    
+
     # Change directory
     ${CD:-"cd"} "$wsp_path" || { printf "FATAL: %s\\n" "Unable to change directory to '$wsp_path'";}
 }

src/providers/github.rs

@@ -1,4 +1,6 @@
-use crate::providers::{create_exclude_regex_set, Provider, APP_USER_AGENT};
+use crate::providers::{
+    create_exclude_regex_set, create_include_regex_set, Provider, APP_USER_AGENT,
+};
 use crate::repository::Repository;
 use anyhow::{bail, Context};
 use console::style;
@@ -50,6 +52,12 @@ pub struct GithubProvider {
     /// Don't clone forked repositories
     skip_forks: bool,
 
+    #[structopt(long = "include")]
+    #[serde(default)]
+    /// Only clone repositories that match these regular expressions. The repository name
+    /// includes the user or organisation name.
+    include: Vec<String>,
+
     #[structopt(long = "auth-http")]
     #[serde(default)]
     /// Use HTTP authentication instead of SSH
@@ -161,6 +169,7 @@ impl Provider for GithubProvider {
 
         let mut after = None;
 
+        let include_regex_set = create_include_regex_set(&self.include)?;
         let exclude_regex_set = create_exclude_regex_set(&self.exclude)?;
 
         // include_forks needs to be None instead of true, as the graphql parameter has three
@@ -224,6 +233,7 @@ impl Provider for GithubProvider {
                     .iter()
                     .map(|r| r.as_ref().unwrap())
                     .filter(|r| !r.is_archived)
+                    .filter(|r| include_regex_set.is_match(&r.name_with_owner))
                     .filter(|r| !exclude_regex_set.is_match(&r.name_with_owner))
                     .map(|repo| self.parse_repo(&self.path, repo)),
             );

src/providers/gitlab.rs

@@ -1,4 +1,6 @@
-use crate::providers::{create_exclude_regex_set, Provider, APP_USER_AGENT};
+use crate::providers::{
+    create_exclude_regex_set, create_include_regex_set, Provider, APP_USER_AGENT,
+};
 use crate::repository::Repository;
 use anyhow::{anyhow, Context};
 use console::style;
@@ -81,6 +83,12 @@ pub struct GitlabProvider {
     /// Environment variable containing the auth token
     env_var: String,
 
+    #[structopt(long = "include")]
+    #[serde(default)]
+    /// Only clone repositories that match these regular expressions. The repository name
+    /// includes the user or organisation name.
+    include: Vec<String>,
+
     #[structopt(long = "auth-http")]
     #[serde(default)]
     /// Use HTTP authentication instead of SSH
@@ -149,6 +157,7 @@ impl Provider for GitlabProvider {
         let mut after = Some("".to_string());
         let name = self.name.to_string().to_lowercase();
 
+        let include_regex_set = create_include_regex_set(&self.include)?;
         let exclude_regex_set = create_exclude_regex_set(&self.exclude)?;
 
         let agent = ureq::AgentBuilder::new()
@@ -210,6 +219,7 @@ impl Provider for GitlabProvider {
                 temp_repositories
                     .into_iter()
                     .filter(|r| !r.archived)
+                    .filter(|r| include_regex_set.is_match(&r.full_path))
                     .filter(|r| !exclude_regex_set.is_match(&r.full_path))
                     .map(|r| {
                         Repository::new(

src/providers/mod.rs

@@ -22,3 +22,12 @@ pub fn create_exclude_regex_set(items: &Vec<String>) -> anyhow::Result<regex::Re
         Ok(regex::RegexSet::new(items).context("Error parsing exclude regular expressions")?)
     }
 }
+
+pub fn create_include_regex_set(items: &Vec<String>) -> anyhow::Result<regex::RegexSet> {
+    if items.is_empty() {
+        let all = vec![".*"];
+        Ok(regex::RegexSet::new(all).context("Error parsing include regular expressions")?)
+    } else {
+        Ok(regex::RegexSet::new(items).context("Error parsing include regular expressions")?)
+    }
+}