diff --git a/cli/src/alluxio.org/cli/cmd/job/cancel.go b/cli/src/alluxio.org/cli/cmd/job/cancel.go new file mode 100644 index 000000000000..bf8b7c8cf0ea --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/cancel.go @@ -0,0 +1,60 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd" + "alluxio.org/cli/env" +) + +var Cancel = &CancelCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cancel", + JavaClassName: cmd.JobShellJavaClass, + }, +} + +type CancelCommand struct { + *env.BaseJavaCommand + jobId int +} + +func (c *CancelCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CancelCommand) ToCommand() *cobra.Command { + command := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Cancel.CommandName, + Short: "Cancels a job asynchronously.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + command.Flags().IntVar(&c.jobId, "id", 0, "Determine a job ID to cancel") + command.MarkFlagRequired("id") + return command +} + +func (c *CancelCommand) Run(args []string) error { + if c.jobId <= 0 { + return stacktrace.NewError("Flag --id should be a positive integer") + } + javaArgs := []string{"cancel", strconv.Itoa(c.jobId)} + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/cmd_status.go b/cli/src/alluxio.org/cli/cmd/job/cmd_status.go new file mode 100644 index 000000000000..63b4c06ebfbd --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/cmd_status.go @@ -0,0 +1,61 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd" + "alluxio.org/cli/env" +) + +var CmdStatus = &CmdStatusCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cmdStatus", + JavaClassName: cmd.JobShellJavaClass, + }, +} + +type CmdStatusCommand struct { + *env.BaseJavaCommand + jobControlId int +} + +func (c *CmdStatusCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CmdStatusCommand) ToCommand() *cobra.Command { + command := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: CmdStatus.CommandName, + Short: "Get the status information for a distributed command.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + command.Flags().IntVar(&c.jobControlId, "id", 0, + "Determine the job control ID to get the status information") + command.MarkFlagRequired("id") + return command +} + +func (c *CmdStatusCommand) Run(args []string) error { + if c.jobControlId <= 0 { + return stacktrace.NewError("Flag --id should be a positive integer") + } + javaArgs := []string{"getCmdStatus", strconv.Itoa(c.jobControlId)} + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/job.go b/cli/src/alluxio.org/cli/cmd/job/job.go new file mode 100644 index 000000000000..a3d902d76a22 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/job.go @@ -0,0 +1,28 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import "alluxio.org/cli/env" + +var Service = &env.Service{ + Name: "job", + Description: "Command line tool for interacting with the job service.", + Commands: []env.Command{ + Cancel, + CmdStatus, + JobStatus, + Leader, + List, + Load, + Submit, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/job/job_status.go b/cli/src/alluxio.org/cli/cmd/job/job_status.go new file mode 100644 index 000000000000..9bbb4e9dd23c --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/job_status.go @@ -0,0 +1,68 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd" + "alluxio.org/cli/env" +) + +var JobStatus = &JobStatusCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "jobStatus", + JavaClassName: cmd.JobShellJavaClass, + }, +} + +type JobStatusCommand struct { + *env.BaseJavaCommand + jobId int + everyTask bool +} + +func (c *JobStatusCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *JobStatusCommand) ToCommand() *cobra.Command { + command := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: JobStatus.CommandName, + Short: "Displays the status info for the specific job.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + command.Flags().IntVar(&c.jobId, "id", 0, + "Determine the job ID to get status info") + command.Flags().BoolVarP(&c.everyTask, "every-task", "v", false, + "Determine display the status of every task") + command.MarkFlagRequired("id") + return command +} + +func (c *JobStatusCommand) Run(args []string) error { + if c.jobId <= 0 { + return stacktrace.NewError("Flag --id should be a positive integer") + } + javaArgs := []string{"stat"} + if c.everyTask { + javaArgs = append(javaArgs, "-v") + } + javaArgs = append(javaArgs, strconv.Itoa(c.jobId)) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/leader.go b/cli/src/alluxio.org/cli/cmd/job/leader.go new file mode 100644 index 000000000000..eab0f5e619a5 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/leader.go @@ -0,0 +1,51 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd" + "alluxio.org/cli/env" +) + +var Leader = &LeaderCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "leader", + JavaClassName: cmd.JobShellJavaClass, + }, +} + +type LeaderCommand struct { + *env.BaseJavaCommand +} + +func (c *LeaderCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *LeaderCommand) ToCommand() *cobra.Command { + command := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Leader.CommandName, + Short: "Prints the hostname of the job master service leader.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return command +} + +func (c *LeaderCommand) Run(args []string) error { + javaArgs := []string{"leader"} + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/list.go b/cli/src/alluxio.org/cli/cmd/job/list.go new file mode 100644 index 000000000000..431c751723bd --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/list.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd" + "alluxio.org/cli/env" +) + +var List = &ListCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "list", + JavaClassName: cmd.JobShellJavaClass, + }, +} + +type ListCommand struct { + *env.BaseJavaCommand +} + +func (c *ListCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ListCommand) ToCommand() *cobra.Command { + command := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: List.CommandName, + Short: "Prints the IDs of the most recent jobs, running and finished, " + + "in the history up to the capacity set in alluxio.job.master.job.capacity", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return command +} + +func (c *ListCommand) Run(args []string) error { + javaArgs := []string{"ls"} + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/load.go b/cli/src/alluxio.org/cli/cmd/job/load.go new file mode 100644 index 000000000000..3f3dc503c24b --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/load.go @@ -0,0 +1,54 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd" + "alluxio.org/cli/env" +) + +var Load = &LoadCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "load", + JavaClassName: cmd.FileSystemShellJavaClass, + }, +} + +type LoadCommand struct { + *env.BaseJavaCommand + path string +} + +func (c *LoadCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *LoadCommand) ToCommand() *cobra.Command { + command := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Load.CommandName, + Short: "Submit load job to Alluxio master, update job options if already exists", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + command.Flags().StringVar(&c.path, "path", "", "Determine the path of the load job to submit") + command.MarkFlagRequired("path") + return command +} + +func (c *LoadCommand) Run(args []string) error { + javaArgs := []string{"load", c.path, "--submit"} + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/submit.go b/cli/src/alluxio.org/cli/cmd/job/submit.go new file mode 100644 index 000000000000..9dd03f323524 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/submit.go @@ -0,0 +1,96 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd" + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Submit = &SubmitCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "submit", + JavaClassName: cmd.FileSystemShellJavaClass, + }, +} + +type SubmitCommand struct { + *env.BaseJavaCommand + operationType string + src string + dst string + activeJobs int + batchSize int +} + +func (c *SubmitCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *SubmitCommand) ToCommand() *cobra.Command { + command := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Submit.CommandName, + Short: "Moves or copies a file or directory in parallel at file level.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + command.Flags().StringVar(&c.operationType, "type", "", "Determine type, options: [cp, mv]") + command.Flags().StringVar(&c.src, "src", "", "The path to move/copy from") + command.Flags().StringVar(&c.dst, "dst", "", "The path to move/copy to") + command.Flags().IntVar(&c.activeJobs, "active-jobs", 3000, + "Number of active jobs that can run at the same time, later jobs must wait") + command.Flags().IntVar(&c.batchSize, "batch-size", 1, "Number of files per request") + command.MarkFlagRequired("type") + command.MarkFlagRequired("src") + command.MarkFlagRequired("dst") + return command +} + +func (c *SubmitCommand) Run(args []string) error { + var javaArgs []string + switch c.operationType { + case "cp": + javaArgs = append(javaArgs, "distributedCp") + if c.activeJobs <= 0 { + return stacktrace.NewError("Flag --active-jobs should be a positive integer") + } + javaArgs = append(javaArgs, "--active-jobs", strconv.Itoa(c.activeJobs)) + if c.batchSize <= 0 { + return stacktrace.NewError("Flag --batch-size should be a positive integer") + } + javaArgs = append(javaArgs, "--batch-size", strconv.Itoa(c.batchSize)) + + case "mv": + javaArgs = append(javaArgs, "distributedMv") + if c.activeJobs != 3000 { + log.Logger.Warningf("Flag --active-jobs is set, but won't be used in this operation type") + } + if c.batchSize != 1 { + log.Logger.Warningf("Flag --batch-size is set, but won't be used in this operation type") + } + + default: + return stacktrace.NewError("Invalid operation type. Must be one of [cp mv].") + } + + javaArgs = append(javaArgs, c.src) + javaArgs = append(javaArgs, c.dst) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/names.go b/cli/src/alluxio.org/cli/cmd/names.go index 732f4791a2ff..7066568adc58 100644 --- a/cli/src/alluxio.org/cli/cmd/names.go +++ b/cli/src/alluxio.org/cli/cmd/names.go @@ -14,4 +14,5 @@ package cmd const ( FileSystemAdminShellJavaClass = "alluxio.cli.fsadmin.FileSystemAdminShell" FileSystemShellJavaClass = "alluxio.cli.fs.FileSystemShell" + JobShellJavaClass = "alluxio.cli.job.JobShell" ) diff --git a/cli/src/alluxio.org/cli/main.go b/cli/src/alluxio.org/cli/main.go index 1e135e32994a..da005e241564 100644 --- a/cli/src/alluxio.org/cli/main.go +++ b/cli/src/alluxio.org/cli/main.go @@ -24,6 +24,7 @@ import ( "alluxio.org/cli/cmd/fs" "alluxio.org/cli/cmd/generate" "alluxio.org/cli/cmd/info" + "alluxio.org/cli/cmd/job" "alluxio.org/cli/cmd/journal" "alluxio.org/cli/cmd/process" "alluxio.org/cli/cmd/quorum" @@ -49,6 +50,7 @@ func main() { fs.Service, generate.Service, info.Service, + job.Service, journal.Service, process.Service, quorum.Service,