diff --git a/cli/src/alluxio.org/cli/cmd/exec/class.go b/cli/src/alluxio.org/cli/cmd/exec/class.go new file mode 100644 index 000000000000..c4d4edfeb892 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/class.go @@ -0,0 +1,70 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Class = &ClassCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "class", + }, +} + +type ClassCommand struct { + *env.BaseJavaCommand + mainClass string + jarFile string + module string +} + +func (c *ClassCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ClassCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "class", + Short: "Run the main method of an Alluxio class.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.jarFile, "jar", "", + "Determine a JAR file to run.") + cmd.Flags().StringVar(&c.module, "m", "", + "Determine a module to run.") + cmd.MarkFlagsMutuallyExclusive("jar", "m") + return cmd +} + +func (c *ClassCommand) Run(args []string) error { + var javaArgs []string + if c.jarFile != "" { + javaArgs = append(javaArgs, "-jar", c.jarFile) + } else if c.module != "" { + javaArgs = append(javaArgs, "-m", c.module) + } else if len(args) != 0 { + c.JavaClassName = args[0] + } else { + return stacktrace.Propagate(nil, "None of JAR, module, nor a java class is specified") + } + + if len(args) > 1 { + javaArgs = append(javaArgs, args[1:]...) + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/exec.go b/cli/src/alluxio.org/cli/cmd/exec/exec.go new file mode 100644 index 000000000000..05152e0e1772 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/exec.go @@ -0,0 +1,29 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "exec", + Description: "Run the main method of an Alluxio class, or end-to-end tests on an Alluxio cluster.", + Commands: []env.Command{ + Class, + TestHdfsMount, + TestHms, + TestJournalCrash, + TestRun, + TestUfs, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_hdfs_mount.go b/cli/src/alluxio.org/cli/cmd/exec/test_hdfs_mount.go new file mode 100644 index 000000000000..441130f4a104 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_hdfs_mount.go @@ -0,0 +1,77 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var TestHdfsMount = &TestHdfsMountCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "hdfsMountTest", + JavaClassName: "alluxio.cli.ValidateHdfsMount", + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console"), + }, +} + +type TestHdfsMountCommand struct { + *env.BaseJavaCommand + path string + readonly bool + shared bool + option string +} + +func (c *TestHdfsMountCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestHdfsMountCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "hdfsMountTest", + Args: cobra.NoArgs, + Short: "Tests runs a set of validations against the given hdfs path.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.path, "path", "", + "specifies the HDFS path you want to validate.") + cmd.Flags().BoolVar(&c.readonly, "readonly", false, + "mount point is readonly in Alluxio.") + cmd.Flags().BoolVar(&c.shared, "shared", false, + "mount point is shared.") + cmd.Flags().StringVar(&c.option, "option", "", + "options associated with this mount point.") + cmd.MarkFlagRequired("path") + return cmd +} + +func (c *TestHdfsMountCommand) Run(args []string) error { + var javaArgs []string + javaArgs = append(javaArgs, c.path) + if c.readonly { + javaArgs = append(javaArgs, "--readonly") + } + if c.shared { + javaArgs = append(javaArgs, "--shared") + } + if c.option != "" { + javaArgs = append(javaArgs, "--option", c.option) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_hms.go b/cli/src/alluxio.org/cli/cmd/exec/test_hms.go new file mode 100644 index 000000000000..0e62b4097ebf --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_hms.go @@ -0,0 +1,75 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "strconv" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var TestHms = &TestHmsCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "hiveMetastoreTest", + JavaClassName: "alluxio.cli.HmsTests", + }, +} + +type TestHmsCommand struct { + *env.BaseJavaCommand + metastore string + database string + tables string + socketTimeout int +} + +func (c *TestHmsCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestHmsCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "hiveMetastoreTest", + Args: cobra.NoArgs, + Short: "Test the configuration, connectivity, and permission of an existing hive metastore.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVarP(&c.metastore, "metastore", "m", "", + "Uri(s) to connect to hive metastore.") + cmd.Flags().StringVarP(&c.database, "database", "d", "default", + "Database to run tests against.") + cmd.Flags().StringVarP(&c.tables, "table", "t", "", + "Tables to run tests against.\n"+ + "Multiple tables should be separated with comma.") + cmd.Flags().IntVarP(&c.socketTimeout, "socketTimeout", "s", -1, + "Socket timeout of hive metastore client in minutes.\n"+ + "Consider increasing this if you have tables with a lot of metadata.") + cmd.MarkFlagRequired("metastore") + return cmd +} + +func (c *TestHmsCommand) Run(args []string) error { + javaArgs := []string{"-m", c.metastore} + if c.database != "" { + javaArgs = append(javaArgs, "-d", c.database) + } + if c.tables != "" { + javaArgs = append(javaArgs, "-t", c.tables) + } + javaArgs = append(javaArgs, "-st", strconv.Itoa(c.socketTimeout)) + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_journal_crash.go b/cli/src/alluxio.org/cli/cmd/exec/test_journal_crash.go new file mode 100644 index 000000000000..bfe077e1a909 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_journal_crash.go @@ -0,0 +1,101 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "fmt" + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var TestJournalCrash = &TestJournalCrashCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "journalCrashTest", + JavaClassName: "alluxio.cli.JournalCrashTest", + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console"), + }, +} + +type TestJournalCrashCommand struct { + *env.BaseJavaCommand + creates int + deletes int + maxAlive int + renames int + testDir string + totalTime int +} + +func (c *TestJournalCrashCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestJournalCrashCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "journalCrashTest", + Args: cobra.NoArgs, + Short: "Test the Master Journal System in a crash scenario.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().IntVar(&c.creates, "creates", 2, + "Number of Client Threads to request create operations.") + cmd.Flags().IntVar(&c.deletes, "deletes", 2, + "Number of Client Threads to request create/delete operations.") + cmd.Flags().IntVar(&c.maxAlive, "maxAlive", 5, + "The maximum time a master should ever be alive during the test, in seconds.") + cmd.Flags().IntVar(&c.renames, "renames", 2, + "Number of Client Threads to request create/rename operations.") + cmd.Flags().StringVar(&c.testDir, "testDir", "/default_tests_files", + "Test Directory on Alluxio.") + cmd.Flags().IntVar(&c.totalTime, "totalTime", 20, + "The total time to run this test, in seconds. This value should be greater than flag --maxAlive.") + return cmd +} + +func (c *TestJournalCrashCommand) Run(args []string) error { + if c.creates <= 0 { + stacktrace.Propagate(nil, "Flag --creates should be a positive integer") + } + if c.deletes <= 0 { + stacktrace.Propagate(nil, "Flag --deletes should be a positive integer") + } + if c.maxAlive <= 0 { + stacktrace.Propagate(nil, "Flag --maxAlive should be a positive integer") + } + if c.renames <= 0 { + stacktrace.Propagate(nil, "Flag --renames should be a positive integer") + } + if c.totalTime <= 0 { + stacktrace.Propagate(nil, "Flag --totalTime should be a positive integer") + } + if c.totalTime < c.maxAlive { + stacktrace.Propagate(nil, "Flag --totalTime should be greater than flag --maxAlive.") + } + javaArgs := []string{ + "-creates", strconv.Itoa(c.creates), + "-deletes", strconv.Itoa(c.deletes), + "-maxAlive", strconv.Itoa(c.maxAlive), + "-renames", strconv.Itoa(c.renames), + "-totalTime", strconv.Itoa(c.totalTime), + } + if c.testDir != "" { + javaArgs = append(javaArgs, "-testDir", c.testDir) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_run.go b/cli/src/alluxio.org/cli/cmd/exec/test_run.go new file mode 100644 index 000000000000..6899c21fac8c --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_run.go @@ -0,0 +1,88 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var TestRun = &TestRunCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "basicIOTest", + JavaClassName: "alluxio.cli.TestRunner", + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console"), + }, +} + +type TestRunCommand struct { + *env.BaseJavaCommand + directory string + operation string + readType string + workers string + writeType string +} + +func (c *TestRunCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestRunCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "basicIOTest", + Args: cobra.NoArgs, + Short: "Run all end-to-end tests, or a specific test, on an Alluxio cluster.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.directory, "directory", "/", + "Alluxio path for the tests working directory. Default: /") + cmd.Flags().StringVar(&c.operation, "operation", "", + "The operation to test, either BASIC or BASIC_NON_BYTE_BUFFER. \n"+ + "By default both operations are tested.") + cmd.Flags().StringVar(&c.readType, "readType", "", + "The read type to use, one of NO_CACHE, CACHE, CACHE_PROMOTE. \n"+ + "By default all readTypes are tested.") + cmd.Flags().StringVar(&c.workers, "workers", "", + "Alluxio worker addresses to run tests on. \n"+ + "If not specified, random ones will be used.") + cmd.Flags().StringVar(&c.writeType, "writeType", "", + "The write type to use, one of MUST_CACHE, CACHE_THROUGH, THROUGH, ASYNC_THROUGH. \n"+ + "By default all writeTypes are tested.") + return cmd +} + +func (c *TestRunCommand) Run(args []string) error { + var javaArgs []string + if c.directory != "" { + javaArgs = append(javaArgs, "--directory", c.directory) + } + if c.operation != "" { + javaArgs = append(javaArgs, "--operation", c.operation) + } + if c.readType != "" { + javaArgs = append(javaArgs, "--readType", c.readType) + } + if c.workers != "" { + javaArgs = append(javaArgs, "--workers", c.workers) + } + if c.writeType != "" { + javaArgs = append(javaArgs, "--writeType", c.writeType) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_ufs.go b/cli/src/alluxio.org/cli/cmd/exec/test_ufs.go new file mode 100644 index 000000000000..1857ef57be20 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_ufs.go @@ -0,0 +1,62 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var TestUfs = &TestUfsCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "ufsTest", + JavaClassName: "alluxio.cli.UnderFileSystemContractTest", + }, +} + +type TestUfsCommand struct { + *env.BaseJavaCommand + path string + test []string +} + +func (c *TestUfsCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestUfsCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "ufsTest", + Args: cobra.NoArgs, + Short: "Test the integration between Alluxio and the given UFS.\n" + + "UFS tests validate the semantics Alluxio expects of the UFS.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.path, "path", "", + "the full UFS path to run tests against.") + cmd.Flags().StringSliceVar(&c.test, "test", nil, + "Test name, this option can be passed multiple times to indicate multiply tests") + cmd.MarkFlagRequired("path") + return cmd +} + +func (c *TestUfsCommand) Run(args []string) error { + javaArgs := []string{"--path", c.path} + for _, singleTest := range c.test { + javaArgs = append(javaArgs, "--test", singleTest) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/main.go b/cli/src/alluxio.org/cli/main.go index 2d9d9afccd26..b761e6410f89 100644 --- a/cli/src/alluxio.org/cli/main.go +++ b/cli/src/alluxio.org/cli/main.go @@ -15,6 +15,7 @@ import ( "os" "alluxio.org/cli/cmd/conf" + "alluxio.org/cli/cmd/exec" "alluxio.org/cli/cmd/fs" "alluxio.org/cli/cmd/generate" "alluxio.org/cli/cmd/info" @@ -39,6 +40,7 @@ func main() { for _, c := range []*env.Service{ conf.Service, + exec.Service, fs.Service, generate.Service, info.Service,