add a option to provide customized bucket endpoint (#1008)

* add a option to provide customized bucket endpoint

* update docs for --bucket

cmd/gateway.go

@@ -221,6 +221,9 @@ func (g *GateWay) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, er
 		}
 		chunkConf.CacheDir = strings.Join(ds, string(os.PathListSeparator))
 	}
+	if c.IsSet("bucket") {
+		format.Bucket = c.String("bucket")
+	}
 	blob, err := createStorage(format)
 	if err != nil {
 		logger.Fatalf("object storage: %s", err)

cmd/mount.go

@@ -206,6 +206,9 @@ func mount(c *cli.Context) error {
 		}
 		chunkConf.CacheDir = strings.Join(ds, string(os.PathListSeparator))
 	}
+	if c.IsSet("bucket") {
+		format.Bucket = c.String("bucket")
+	}
 	blob, err := createStorage(format)
 	if err != nil {
 		logger.Fatalf("object storage: %s", err)
@@ -303,6 +306,10 @@ func clientFlags() []cli.Flag {
 		defaultCacheDir = path.Join(homeDir, ".juicefs", "cache")
 	}
 	return []cli.Flag{
+		&cli.StringFlag{
+			Name:  "bucket",
+			Usage: "customized endpoint to access object store",
+		},
 		&cli.IntFlag{
 			Name:  "get-timeout",
 			Value: 60,

docs/en/command_reference.md

@@ -188,6 +188,9 @@ dir entry cache timeout in seconds (default: 1)
 `--enable-xattr`<br />
 enable extended attributes (xattr) (default: false)
 
+`--bucket`<br />
+customized endpoint to access object store
+
 `--get-timeout value`<br />
 the max number of seconds to download an object (default: 60)
 

docs/en/hadoop_java_sdk.md

@@ -149,6 +149,7 @@ Please refer to the following table to set the relevant parameters of the JuiceF
 
 | Configuration             | Default Value | Description                                                  |
 | ------------------------- | ------------- | ------------------------------------------------------------ |
+| `juicefs.bucket`          |               | Specify a different endpoint for object storage              |
 | `juicefs.debug`           | `false`       | Whether enable debug log                                     |
 | `juicefs.access-log`      |               | Access log path. Ensure Hadoop application has write permission, e.g. `/tmp/juicefs.access.log`. The log file will rotate  automatically to keep at most 7 files. |
 | `juicefs.superuser`       | `hdfs`        | The super user                                               |

docs/en/how_to_setup_object_storage.md

@@ -7,8 +7,8 @@ By reading [JuiceFS Technical Architecture](architecture.md) and [How JuiceFS St
 When creating a JuiceFS file system, setting up data storage generally involves the following options:
 
 - `--storage` Specify the storage service to be used by the file system, e.g. `--storage s3`
-- `--bucket` Specify the bucket endpoint of the object storage in a specific format, e.g. `--bucket https://myjuicefs.s3.us-east-2.amazonaws.com`
-- `--access-key` and `--secret-key` is the authentication key used when accessing the object storage service. You need to create it on the corresponding cloud platform.
+- `--bucket` Specify the bucket endpoint of the object storage in a specific format, e.g. `--bucket https://myjuicefs.s3.us-east-2.amazonaws.com`. If the object storage uses different endpoint in different environment, it could be specified by `--bucket` of mount command.
+- `--access-key` and `--secret-key` is the authentication key used when accessing the object storage service. You need to create it on the corresponding cloud platform. When the object storage can be accessed based on other authentication methods, these can be left empty.
 
 For example, the following command uses Amazon S3 object storage to create a file system:
 

docs/zh_cn/command_reference.md

@@ -188,6 +188,9 @@ consul注册中心地址(默认: "127.0.0.1:8500")
 `--enable-xattr`<br />
 启用扩展属性 (xattr) 功能 (默认: false)
 
+`--bucket`<br />
+为当前挂载点指定访问访对象存储的 endpoint (默认: 无)
+
 `--get-timeout value`<br />
 下载一个对象的超时时间；单位为秒 (默认: 60)
 

docs/zh_cn/hadoop_java_sdk.md

@@ -149,6 +149,7 @@ $ make win
 
 | 配置项                    | 默认值  | 描述                                                         |
 | ------------------------- | ------- | ------------------------------------------------------------ |
+| `juicefs.bucket`          |         | 为对象存储指定跟格式化时不同的访问地址                             |
 | `juicefs.debug`           | `false` | 是否开启 debug 日志                                          |
 | `juicefs.access-log`      |         | 访问日志的路径。需要所有应用都有写权限，可以配置为 `/tmp/juicefs.access.log`。该文件会自动轮转，保留最近 7 个文件。 |
 | `juicefs.superuser`       | `hdfs`  | 超级用户                                                     |

docs/zh_cn/how_to_setup_object_storage.md

@@ -7,8 +7,8 @@
 在创建 JuiceFS 文件系统时，设置数据存储一般涉及以下几个选项：
 
 - `--storage` 指定文件系统要使用的存储服务，例如：`--storage s3`
-- `--bucket` 按特定格式指定对象存储的 bucket 地址，例如：`--bucket https://myjuicefs.s3.us-east-2.amazonaws.com`
-- `--access-key` 和 `--secret-key` 用来指定访问对象存储的身份认证密钥，需要在相应云平台上创建。
+- `--bucket` 按特定格式指定对象存储的 bucket 地址，例如：`--bucket https://myjuicefs.s3.us-east-2.amazonaws.com`。当对象存储在不同的网络环境需要使用不同的地址时，可以在挂载时通过 `--bucket`参数为当前挂载点指定跟格式化时不一样的地址。
+- `--access-key` 和 `--secret-key` 用来指定访问对象存储的身份认证密钥，需要在相应云平台上创建。如果使用的机器上已经通过环境变量或者其他方式获得了访问对象存储的权限，它们可以不提供。
 
 例如，以下命令使用 Amazon S3 对象存储创建文件系统：
 

sdk/java/libjfs/main.go

@@ -214,6 +214,7 @@ func freeHandle(fd int) {
 
 type javaConf struct {
 	MetaURL         string  `json:"meta"`
+	Bucket          string  `json:"bucket"`
 	ReadOnly        bool    `json:"readOnly"`
 	OpenCache       float64 `json:"openCache"`
 	BackupMeta      int64   `json:"backupMeta"`
@@ -362,6 +363,9 @@ func jfs_init(cname, jsonConf, user, group, superuser, supergroup *C.char) uintp
 			go metric.UpdateMetrics(m)
 		}
 
+		if jConf.Bucket != "" {
+			format.Bucket = jConf.Bucket
+		}
 		blob, err := createStorage(format)
 		if err != nil {
 			logger.Fatalf("object storage: %s", err)

sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java

@@ -310,6 +310,7 @@ public void initialize(URI uri, Configuration conf) throws IOException {
     for (String key : bkeys) {
       obj.put(key, Boolean.valueOf(getConf(conf, key, "false")));
     }
+    obj.put("bucket", getConf(conf, "bucket", ""));
     obj.put("readOnly", Boolean.valueOf(getConf(conf, "read-only", "false")));
     obj.put("cacheDir", getConf(conf, "cache-dir", "memory"));
     obj.put("cacheSize", Integer.valueOf(getConf(conf, "cache-size", "100")));
@@ -503,11 +504,11 @@ public static Libjfs loadLibrary() throws IOException {
 
     LibraryLoader<Libjfs> libjfsLibraryLoader = LibraryLoader.create(Libjfs.class);
     libjfsLibraryLoader.failImmediately();
-    String name = "libjfs.3.so";
+    String name = "libjfs.4.so";
     File dir = new File("/tmp");
     String os = System.getProperty("os.name");
     if (os.toLowerCase().contains("windows")) {
-      name = "libjfs2.dll";
+      name = "libjfs3.dll";
       dir = new File(System.getProperty("java.io.tmpdir"));
     }
     File libFile = new File(dir, name);