Merge pull request #2 from thalesmg/tmg-fixes-1

various fixes

.github/workflows/ci.yaml

@@ -33,6 +33,12 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-hex-
       - name: build
-        run: rebar3 compile
+        run: make compile
       - name: eunit
-        run: rebar3 eunit -m erlazure_xml_tests,erlazure_utils_tests,erlazure_queue_tests,erlazure_blob_tests
+        run: make eunit
+      - name: start azurite
+        run: test/start_azurite.sh
+      - name: ct
+        run: make ct
+        env:
+          AZURITE_ENDPOINT: "http://127.0.0.1:10000/"

Makefile

@@ -0,0 +1,12 @@
+REBAR ?= rebar3
+
+compile:
+	$(REBAR) compile
+
+.PHONY: eunit
+eunit:
+	$(REBAR) eunit -v -m erlazure,erlazure_xml_tests,erlazure_utils_tests,erlazure_queue_tests,erlazure_blob_tests
+
+.PHONY: ct
+ct:
+	$(REBAR) ct -v --readable=true

README.md

@@ -23,7 +23,7 @@ Erlazure requires OTP version R16+.
   * Delete message
   * Clear messages
   * Update message
-  
+
 * Blob storage service
   * List containers
   * Create container
@@ -39,7 +39,7 @@ Erlazure requires OTP version R16+.
   * Put block list
   * Get block list
   * Lease container
-  
+
 * Table storage service
   * List tables
   * New table
@@ -53,6 +53,16 @@ Start an instance of erlazure by calling ```erlazure:start/2``` where **Account*
 ```
 Account and Key are strings.
 
+### Using an emulated service like [Azurite](https://github.com/Azure/Azurite/blob/2bb552e703772b9a57ca713ef271c3c7c624a535/README.md)
+
+```erlang
+%% default dev credentials from Azurite
+Account = "devstoreaccount1".
+Key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==".
+%% Mind the trailing slash at the end of the endpoint.
+{ok, Pid} = erlazure:start(#{account => Account, key => Key, endpoint => "http://127.0.0.1:10000/"})
+```
+
 ## Calling Azure services
 Almost each azure services request has three corresponding functions in ```erlazure``` module, the first has minimal set of parameters, the second has additionaly list of ```Options``` and the third has additionaly ```Timeout``` parameter.
 

test/erlazure_SUITE.erl

@@ -0,0 +1,127 @@
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Redistribution and use in source and binary forms, with or without
+%% modification, are permitted provided that the following conditions are met:
+%%
+%% * Redistributions of source code must retain the above copyright notice,
+%% this list of conditions and the following disclaimer.
+%% * Redistributions in binary form must reproduce the above copyright
+%% notice, this list of conditions and the following disclaimer in the
+%% documentation and/or other materials provided with the distribution.
+%% * Neither the name of erlazure nor the names of its contributors may be used to
+%% endorse or promote products derived from this software without specific
+%% prior written permission.
+%%
+%% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+%% AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+%% IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+%% ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+%% LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+%% CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+%% SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+%% INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+%% CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+%% ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+%% POSSIBILITY OF SUCH DAMAGE.
+-module(erlazure_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include("erlazure.hrl").
+
+%%------------------------------------------------------------------------------
+%% Type definitions
+%%------------------------------------------------------------------------------
+
+%% Default Azurite credentials
+%% See: https://github.com/Azure/Azurite/blob/main/README.md#default-storage-account
+-define(ACCOUNT, "devstoreaccount1").
+-define(KEY, "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==").
+
+%%------------------------------------------------------------------------------
+%% CT boilerplate
+%%------------------------------------------------------------------------------
+
+all() ->
+    test_utils:all(?MODULE).
+
+init_per_suite(Config) ->
+    Endpoint = os:getenv("AZURITE_ENDPOINT", "http://127.0.0.1:10000/"),
+    #{host := Host, port := Port} = uri_string:parse(Endpoint),
+    case test_utils:is_tcp_server_available(Host, Port) of
+        false ->
+            throw(endpoint_unavailable);
+        true ->
+            ok
+    end,
+    [{endpoint, Endpoint} | Config].
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(_TestCase, Config) ->
+    delete_all_containers(Config),
+    ok.
+
+%%------------------------------------------------------------------------------
+%% Helper fns
+%%------------------------------------------------------------------------------
+
+start(Config) ->
+    Endpoint = ?config(endpoint, Config),
+    {ok, Pid} = erlazure:start(#{account => ?ACCOUNT, key => ?KEY, endpoint => Endpoint}),
+    Pid.
+
+delete_all_containers(Config) ->
+    Pid = start(Config),
+    {Containers, _} = erlazure:list_containers(Pid),
+    lists:foreach(
+      fun(#blob_container{name = Name}) ->
+        {ok, deleted} = erlazure:delete_container(Pid, Name)
+      end,
+      Containers).
+
+container_name(Name) ->
+    IOList = re:replace(atom_to_list(Name), <<"[^a-z0-9-]">>, <<"-">>, [global]),
+    binary_to_list(iolist_to_binary(IOList)).
+
+%%------------------------------------------------------------------------------
+%% Test cases : blob storage
+%%------------------------------------------------------------------------------
+
+%% Basic smoke test for basic blob storage operations.
+t_blob_storage_smoke_test(Config) ->
+    Endpoint = ?config(endpoint, Config),
+    {ok, Pid} = erlazure:start(#{account => ?ACCOUNT, key => ?KEY, endpoint => Endpoint}),
+    %% Create a container
+    Container = container_name(?FUNCTION_NAME),
+    ?assertMatch({[], _}, erlazure:list_containers(Pid)),
+    ?assertMatch({ok, created}, erlazure:create_container(Pid, Container)),
+    %% Upload some blobs
+    ?assertMatch({ok, created}, erlazure:put_block_blob(Pid, Container, "blob1", <<"1">>)),
+    ?assertMatch({ok, created}, erlazure:put_block_blob(Pid, Container, "blob2", <<"2">>)),
+    ?assertMatch({[#cloud_blob{name = "blob1"}, #cloud_blob{name = "blob2"}], _},
+                 erlazure:list_blobs(Pid, Container)),
+    %% Read back data
+    ?assertMatch({ok, <<"1">>}, erlazure:get_blob(Pid, Container, "blob1")),
+    ?assertMatch({ok, <<"2">>}, erlazure:get_blob(Pid, Container, "blob2")),
+    %% Delete blob
+    ?assertMatch({ok, deleted}, erlazure:delete_blob(Pid, Container, "blob1")),
+    ?assertMatch({[#cloud_blob{name = "blob2"}], _},
+                 erlazure:list_blobs(Pid, Container)),
+    %% Delete container
+    ?assertMatch({ok, deleted}, erlazure:delete_container(Pid, Container)),
+    ok.
+
+%% Basic smoke test to check that we can pass already wrapped keys to `erlazure:start`.
+t_blob_storage_wrapped_key(Config) ->
+    Endpoint = ?config(endpoint, Config),
+    {ok, Pid} = erlazure:start(#{account => ?ACCOUNT, key => ?KEY, endpoint => Endpoint}),
+    ?assertMatch({[], _}, erlazure:list_containers(Pid)),
+    ok.

test/start_azurite.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+docker run -d --rm -p 10000:10000 --name azurite mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 -d debug.log

test/test_utils.erl

@@ -28,8 +28,38 @@
 -module(test_utils).
 -author("Dmitry Kataskin").
 
-%% API
--export([append_ticks/1, get_ticks/0, read_file/1]).
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-define(DEFAULT_TCP_SERVER_CHECK_AVAIL_TIMEOUT, 1000).
+
+all(Module) ->
+    lists:usort([
+        F
+     || {F, 1} <- Module:module_info(exports),
+        string:substr(atom_to_list(F), 1, 2) == "t_"
+    ]).
+
+-spec is_tcp_server_available(
+    Host :: inet:socket_address() | inet:hostname(),
+    Port :: inet:port_number()
+) -> boolean.
+is_tcp_server_available(Host, Port) ->
+    is_tcp_server_available(Host, Port, ?DEFAULT_TCP_SERVER_CHECK_AVAIL_TIMEOUT).
+
+-spec is_tcp_server_available(
+    Host :: inet:socket_address() | inet:hostname(),
+    Port :: inet:port_number(),
+    Timeout :: integer()
+) -> boolean.
+is_tcp_server_available(Host, Port, Timeout) ->
+    case gen_tcp:connect(Host, Port, [], Timeout) of
+        {ok, Socket} ->
+            gen_tcp:close(Socket),
+            true;
+        {error, _} ->
+            false
+    end.
 
 append_ticks(Name) ->
         Name ++ integer_to_list(get_ticks()).
@@ -43,4 +73,4 @@ read_file(FileName) ->
         erlang:binary_to_list(Binary).
 
 file_path(File) ->
-        filename:join([code:priv_dir(erlazure), responses, File]).
+        filename:join([code:priv_dir(erlazure), responses, File]).