Scnm

.env

@@ -1,7 +1,18 @@
-DATASET_PATH=/path/to/datasets
-GMT_PATH=./genequery
+DATASET_PATH=/home/maksim/box_dir
+GMT_PATH=/home/maksim/scnm/gmts
 MONGODB_HOST=mongodb://mongo:27017
 MONGODB_DATABASE=scn
 MONGODB_COLLECTION=datasets
 PROD_PATH=/scn/scn_js/prod
 DOCS_PATH=/scn_docs/build/docs
+TMP_PATH=/tmp/scnm_temp
+BOX_SOURCE=scn_m_datasets
+BOX_CACHE=/home/maksim/box_cache
+BOX_WEBHOOK_KEY_PRIMARY=key1
+BOX_WEBHOOK_KEY_SECONDARY=key2
+BOX_CLIENT_ID=key3
+BOX_CLIENT_SECRET=key4
+BOX_ENTERPRISE_ID=some_id
+
+
+

docker-compose.yml

@@ -10,20 +10,19 @@ services:
       - "./nginx/sites-enabled:/etc/nginx/sites-enabled"
       - "./scn_js:/scn/scn_js"
       - "./scn_docs/build/:/scn/docs"
-      - "$DATASET_PATH:/var/datasets"
+      - "$DATASET_PATH:/var/datasets:shared"
     ports:
-      - "31:80"
+      - "3133:80"
 
   genequery:
-    container_name: gq
     depends_on:
       - fs
     restart: always
     build: ./genequery
     expose:
       - 8423
     ports:
-      - "8423:8423"
+      - "8433:8423"
     volumes:
       - "$GMT_PATH:/genequery/data-files"
 
@@ -32,11 +31,12 @@ services:
     image: mongo:latest
     restart: always
 #    ports:
-#      - "27017:27017"
+#      - "27033:27017"
+    depends_on:
+       - rclone_fs
     volumes:
       - "./mongo/init-mongo.js:/docker-entrypoint-initdb.d/init-mongo.js:ro"
-      - "$DATASET_PATH:/var/datasets"
-
+      - "$DATASET_PATH:/var/datasets:shared"
   fs:
     env_file:
       - .env
@@ -45,9 +45,11 @@ services:
       - mongo
     build: ./scn_fs
     volumes:
-      - "$DATASET_PATH:/var/datasets"
+      - "$DATASET_PATH:/var/datasets:shared"
+      - "$BOX_CACHE:/data/cache:shared"
       - "$GMT_PATH:/vat/gmt"
-    command: ["java", "-Djava.library.path=./", "-jar", "fs.jar", "/var/datasets", "/vat/gmt"]
+      - "$TMP_PATH:/var/tmp"
+    command: ["java", "-Djava.library.path=./", "-jar", "fs.jar", "/var/datasets", "/vat/gmt", "All Files/$BOX_SOURCE", "$BOX_WEBHOOK_KEY_PRIMARY", "$BOX_WEBHOOK_KEY_SECONDARY", "$BOX_ENTERPRISE_ID","$BOX_CLIENT_ID", "$BOX_CLIENT_SECRET"]
 
   web:
     env_file:
@@ -57,6 +59,44 @@ services:
       - fs
     build: ./scn_server
     volumes:
-      - "$DATASET_PATH:/var/datasets"
+      - "$DATASET_PATH:/var/datasets:shared"
+      - "$BOX_CACHE:/data/cache:shared"
       - "./scn_js:/scn/scn_js"
     command: ["java", "-Djava.library.path=./", "-jar", "server.jar"]
+
+  rclone_fs:
+      env_file:
+        - .env
+      restart: always
+      image:  rclone/rclone
+      user: 1014:1014
+      volumes:
+        - "~/.config/rclone:/config/rclone"
+        - "$DATASET_PATH:/data/box:shared"
+        - "$BOX_CACHE:/data/cache:shared"
+        - "/etc/passwd:/etc/passwd:ro"
+        - "/etc/group:/etc/group:ro"
+      ports:
+        - "5533:5572"
+      devices:
+        - "/dev/fuse:/dev/fuse"
+      cap_add:
+        - SYS_ADMIN
+      security_opt:
+        - apparmor:unconfined
+      command: ["mount", "remote:/$BOX_SOURCE", "/data/box",
+    "--vfs-cache-mode", "full",
+    "--cache-dir", "/data/cache",
+    "--vfs-cache-max-age", "10h",
+    "--vfs-cache-poll-interval", "120s",
+    "--vfs-read-ahead", "20M",
+    "--buffer-size", "2000M",
+    "--vfs-disk-space-total-size", "10G",
+    "--read-only",
+    "--dir-cache-time", "5m",
+    "--poll-interval", "60s",
+    "--allow-non-empty",
+    "--allow-other",
+    "-v",
+    "--rc",
+    "--rc-addr", ":5572"]

nginx/sites-enabled/scn.conf

@@ -89,6 +89,13 @@ server {
         proxy_cache_valid 200 60m;
         proxy_pass http://web:8080;
     }
+    location /scn_fs/ {
+	    proxy_cache_revalidate on;
+        proxy_cache my_cache;
+        proxy_cache_key $host$uri$is_args$args;
+        proxy_cache_valid 200 60m;
+        proxy_pass http://fs:8081;
+    }
 
     location /perform-enrichment {
         proxy_pass http://genequery:8423;

scn_fs/build.gradle.kts

@@ -25,6 +25,7 @@ repositories {
 
 dependencies {
     val mongoVersion: String by System.getProperties()
+    val ktorVersion: String by System.getProperties()
     implementation(kotlin("stdlib"))
     implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2")
     implementation("org.jetbrains.kotlinx:kotlinx-datetime-jvm:0.3.0")
@@ -37,6 +38,16 @@ dependencies {
     testImplementation("org.jetbrains.kotlin:kotlin-test")
     testImplementation("junit", "junit", "4.12")
     testImplementation(kotlin("script-runtime"))
+    implementation("io.ktor:ktor-server-netty:$ktorVersion")
+    implementation("io.ktor:ktor-server-core:$ktorVersion")
+    implementation("io.ktor:ktor-jackson:$ktorVersion")
+    implementation("io.ktor:ktor-server-host-common:$ktorVersion")
+    implementation("io.ktor:ktor-jackson:$ktorVersion")
+    implementation("io.ktor:ktor-client-core:$ktorVersion")
+    implementation("io.ktor:ktor-client-core-jvm:$ktorVersion")
+    implementation("io.ktor:ktor-client-apache:$ktorVersion")
+    implementation("io.ktor:ktor-gson:$ktorVersion")
+    implementation("com.box:box-java-sdk:4.4.0")
 }
 
 sourceSets {