service_test_s3.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Service Test S3

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
    paths:
      - "core/src/**"
      - "core/tests/**"
      - "!core/src/docs/**"
      - "!core/src/services/**"
      - "core/src/services/s3/**"
      - "bindings/java/**"
      - ".github/workflows/service_test_s3.yml"
      - "fixtures/s3/**"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

jobs:
  aws_s3:
    runs-on: ubuntu-latest
    if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
    steps:
      - uses: actions/checkout@v3
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup
        with:
          need-nextest: true
      - name: Test
        shell: bash
        working-directory: core
        run: cargo nextest run s3
        env:
          OPENDAL_S3_TEST: ${{ secrets.OPENDAL_S3_TEST }}
          OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }}
          OPENDAL_S3_BUCKET: ${{ secrets.OPENDAL_S3_BUCKET }}
          OPENDAL_S3_ENDPOINT: ${{ secrets.OPENDAL_S3_ENDPOINT }}
          OPENDAL_S3_ACCESS_KEY_ID: ${{ secrets.OPENDAL_S3_ACCESS_KEY_ID }}
          OPENDAL_S3_SECRET_ACCESS_KEY: ${{ secrets.OPENDAL_S3_SECRET_ACCESS_KEY }}
          OPENDAL_S3_REGION: ap-northeast-1

  aws_s3_with_virtual_host:
    runs-on: ubuntu-latest
    if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
    steps:
      - uses: actions/checkout@v3
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup
        with:
          need-nextest: true
      - name: Test
        shell: bash
        working-directory: core
        run: cargo nextest run s3
        env:
          OPENDAL_S3_TEST: ${{ secrets.OPENDAL_S3_TEST }}
          OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }}
          OPENDAL_S3_BUCKET: ${{ secrets.OPENDAL_S3_BUCKET }}
          OPENDAL_S3_ENDPOINT: ${{ secrets.OPENDAL_S3_ENDPOINT }}
          OPENDAL_S3_ACCESS_KEY_ID: ${{ secrets.OPENDAL_S3_ACCESS_KEY_ID }}
          OPENDAL_S3_SECRET_ACCESS_KEY: ${{ secrets.OPENDAL_S3_SECRET_ACCESS_KEY }}
          OPENDAL_S3_ENABLE_VIRTUAL_HOST_STYLE: on
          OPENDAL_S3_REGION: ap-northeast-1

  aws_s3_with_sse_c:
    runs-on: ubuntu-latest
    if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
    steps:
      - uses: actions/checkout@v3
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup
        with:
          need-nextest: true
      - name: Test
        shell: bash
        working-directory: core
        run: cargo nextest run s3
        env:
          OPENDAL_S3_TEST: ${{ secrets.OPENDAL_S3_TEST }}
          OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }}
          OPENDAL_S3_BUCKET: ${{ secrets.OPENDAL_S3_BUCKET }}
          OPENDAL_S3_ENDPOINT: ${{ secrets.OPENDAL_S3_ENDPOINT }}
          OPENDAL_S3_ACCESS_KEY_ID: ${{ secrets.OPENDAL_S3_ACCESS_KEY_ID }}
          OPENDAL_S3_SECRET_ACCESS_KEY: ${{ secrets.OPENDAL_S3_SECRET_ACCESS_KEY }}
          OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: AES256
          OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY: MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=
          OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5: zZ5FnqcIqUjVwvWmyog4zw==
          OPENDAL_S3_REGION: ap-northeast-1

  aws_s3_with_assume_role:
    runs-on: ubuntu-latest
    if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
    permissions:
      id-token: write
      contents: read
    steps:
      - uses: actions/checkout@v3
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup
        with:
          need-nextest: true
      - uses: actions/github-script@v6
        id: id-token
        with:
          script: return await core.getIDToken("sts.amazonaws.com")
          result-encoding: string
      - name: Write ID token to file
        run: echo "${{ steps.id-token.outputs.result }}" > core/tests/data/web_identity_token
      - name: Test
        shell: bash
        working-directory: core
        run: cargo nextest run s3
        env:
          AWS_WEB_IDENTITY_TOKEN_FILE: tests/data/web_identity_token
          AWS_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing-assume
          OPENDAL_S3_TEST: on
          OPENDAL_S3_ROOT: CI/
          OPENDAL_S3_BUCKET: opendal-testing
          OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing
          OPENDAL_S3_REGION: ap-northeast-1

  minio_s3:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Setup MinIO Server
        shell: bash
        working-directory: fixtures/s3
        run: docker-compose -f docker-compose-minio.yml up -d
      - name: Setup test bucket
        env:
          AWS_ACCESS_KEY_ID: "minioadmin"
          AWS_SECRET_ACCESS_KEY: "minioadmin"
          AWS_EC2_METADATA_DISABLED: "true"
        run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup
        with:
          need-nextest: true
      - name: Test
        shell: bash
        working-directory: core
        run: cargo nextest run s3
        env:
          OPENDAL_S3_TEST: on
          OPENDAL_S3_BUCKET: test
          OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000"
          OPENDAL_S3_ACCESS_KEY_ID: minioadmin
          OPENDAL_S3_SECRET_ACCESS_KEY: minioadmin
          OPENDAL_S3_REGION: us-east-1

  anonymous_minio_s3:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Setup MinIO Server
        shell: bash
        working-directory: fixtures/s3
        run: |
          docker-compose -f docker-compose-minio.yml up -d
      - name: Setup test bucket
        env:
          AWS_ACCESS_KEY_ID: "minioadmin"
          AWS_SECRET_ACCESS_KEY: "minioadmin"
          AWS_EC2_METADATA_DISABLED: "true"
        run: |
          aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test

          curl -O https://dl.min.io/client/mc/release/linux-amd64/mc
          chmod +x mc
          ./mc alias set local http://127.0.0.1:9000/ minioadmin minioadmin
          ./mc anonymous set public local/test

      - name: Setup Rust toolchain
        uses: ./.github/actions/setup
        with:
          need-nextest: true
      - name: Test
        shell: bash
        working-directory: core
        run: cargo nextest run s3
        env:
          OPENDAL_S3_TEST: on
          OPENDAL_S3_BUCKET: test
          OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000"
          OPENDAL_S3_ALLOW_ANONYMOUS: on
          OPENDAL_S3_REGION: us-east-1

  r2:
    runs-on: ubuntu-latest
    if: (github.event_name == 'push' && github.repository == 'apache/incubator-opendal') || !github.event.pull_request.head.repo.fork
    steps:
      - uses: actions/checkout@v3
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup
        with:
          need-nextest: true

      - name: Load secret
        id: op-load-secret
        uses: 1password/load-secrets-action@v1
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
          OPENDAL_S3_TEST: op://services/r2/test
          OPENDAL_S3_BUCKET: op://services/r2/bucket
          OPENDAL_S3_ENDPOINT: op://services/r2/endpoint
          OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id
          OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key

      - name: Test
        shell: bash
        working-directory: core
        run: cargo nextest run s3
        env:
          OPENDAL_S3_REGION: auto
          # This is the R2's limitation
          # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
          OPENDAL_S3_BATCH_MAX_OPERATIONS: 700
          # This is the R2's limitation
          # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
          OPENDAL_S3_ENABLE_EXACT_BUF_WRITE: true

  java:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Setup MinIO Server
        shell: bash
        working-directory: fixtures/s3
        run: docker-compose -f docker-compose-minio.yml up -d
      - name: Setup test bucket
        env:
          AWS_ACCESS_KEY_ID: "minioadmin"
          AWS_SECRET_ACCESS_KEY: "minioadmin"
          AWS_EC2_METADATA_DISABLED: "true"
        run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup
      - name: Test
        shell: bash
        working-directory: bindings/java
        run: ./mvnw test -Dtest=org.apache.opendal.test.behavior.S3Test
        env:
          OPENDAL_S3_TEST: on
          OPENDAL_S3_BUCKET: test
          OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000"
          OPENDAL_S3_ACCESS_KEY_ID: minioadmin
          OPENDAL_S3_SECRET_ACCESS_KEY: minioadmin
          OPENDAL_S3_REGION: us-east-1