Skip to content

Commit

Permalink
Refactor of top level tests (#484)
Browse files Browse the repository at this point in the history 
* Example cross cloud delivery test

* Test cases for cross cloud delivery

- azure -> s3
- s3 -> azure
- file -> s3

* Disable UI if no port is provided

* Pre configure minio with test-bucket

* Run simulation tests in ci

* Integration test documentation
  • Loading branch information
@whytheplatypus
whytheplatypus authored Sep 30, 2024
1 parent 06542fa commit 03f3aae
Show file tree
Hide file tree
Showing 13 changed files with 239 additions and 119 deletions.
16 changes: 13 additions & 3 deletions .github/workflows/tus-upload-server-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,22 @@ jobs:
run: go tool cover -func=c.out
- name: Build Check
run: go build -o upload-server ./cmd/main.go
simulation-test:
name: Simulation Test
runs-on: ubuntu-latest
env:
AZURITE_STORAGE_KEY: ${{ secrets.AZURITE_STORAGE_KEY }}
steps:
- uses: actions/checkout@v4
- name: Install podman compose
run: pip3 install podman-compose
- name: Run Tests
run: podman-compose -f docker-compose.yml -f docker-compose.azurite.yml -f docker-compose.minio.yml -f docker-compose.testing.yml up --exit-code-from upload-server
- name: Show coverage
run: go tool cover -func=c.out
run-fortify-scan:
uses: ./.github/workflows/remote-cd-trigger-template.yml
with:
WORKFLOW: fortify.yml
REF: ${{ github.head_ref }}
secrets: inherit



28 changes: 8 additions & 20 deletions upload-server/cmd/cli/logger.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,38 +3,27 @@ package cli
import (
"log/slog"
"os"
"reflect"
"strings"

expslog "golang.org/x/exp/slog"

"github.com/cdcgov/data-exchange-upload/upload-server/internal/appconfig"
"github.com/cdcgov/data-exchange-upload/upload-server/pkg/sloger"
)

var (
logger *slog.Logger
)

func init() {
type Empty struct{}
pkgParts := strings.Split(reflect.TypeOf(Empty{}).PkgPath(), "/")
// add package name to app logger
logger = sloger.With("pkg", pkgParts[len(pkgParts)-1])
}

// AppLogger, this is the custom application logger for uniformity
func AppLogger(appConfig appconfig.AppConfig) *slog.Logger {

// Configure debug on if needed, otherwise should be off
var opts *slog.HandlerOptions
opts := &slog.HandlerOptions{
AddSource: true,
} // .opts

if appConfig.LoggerDebugOn {
opts.Level = slog.LevelDebug

opts = &slog.HandlerOptions{
Level: slog.LevelDebug,
AddSource: true,
} // .opts
} // .if

logger := slog.New(slog.NewJSONHandler(os.Stdout, opts))
Expand All @@ -57,14 +46,13 @@ func AppLogger(appConfig appconfig.AppConfig) *slog.Logger {
func ExpAppLogger(appConfig appconfig.AppConfig) *expslog.Logger {

// Configure debug on if needed, otherwise should be off
var opts *expslog.HandlerOptions
opts := &expslog.HandlerOptions{
AddSource: true,
} // .opts

if appConfig.LoggerDebugOn {
opts.Level = expslog.LevelDebug

opts = &expslog.HandlerOptions{
Level: expslog.LevelDebug,
AddSource: true,
} // .opts
} // .if

logger := expslog.New(expslog.NewJSONHandler(os.Stdout, opts))
Expand Down
100 changes: 36 additions & 64 deletions upload-server/cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,50 +7,26 @@ import (
"net/http"
"os"
"os/signal"
"runtime/debug"
"sync"
"syscall"
"testing"
"time"

"github.com/cdcgov/data-exchange-upload/upload-server/internal/event"
"github.com/cdcgov/data-exchange-upload/upload-server/internal/metrics"
"github.com/cdcgov/data-exchange-upload/upload-server/internal/postprocessing"
"github.com/cdcgov/data-exchange-upload/upload-server/internal/ui"
"github.com/cdcgov/data-exchange-upload/upload-server/pkg/reports"
"github.com/cdcgov/data-exchange-upload/upload-server/pkg/sloger"
"github.com/cdcgov/data-exchange-upload/upload-server/pkg/slogerxexp"

"github.com/cdcgov/data-exchange-upload/upload-server/cmd/cli"
"github.com/cdcgov/data-exchange-upload/upload-server/internal/appconfig"
"github.com/cdcgov/data-exchange-upload/upload-server/pkg/sloger"
"github.com/cdcgov/data-exchange-upload/upload-server/pkg/slogerxexp"
"github.com/joho/godotenv"
) // .import

const appMainExitCode = 1

var (
appConfig appconfig.AppConfig
logger *slog.Logger
)

// NOTE: this large init file may be an antipattern.
// A main reason for it is to enable to cross cutting logging aspect.
// If another way is found to manage that this should be moved to main.
func init() {
ctx := context.Background()

buildInfo, _ := debug.ReadBuildInfo()
logInfo := []any{"buildInfo.Main.Path", buildInfo.Main.Path}
slog.With(logInfo...)
// ------------------------------------------------------------------
// parse and load cli flags
// ------------------------------------------------------------------
if !testing.Testing() {
if err := cli.ParseFlags(); err != nil {
slog.Error("error starting app, error parsing cli flags", "error", err)
os.Exit(appMainExitCode)
} // .if
}

if cli.Flags.AppConfigPath != "" {
slog.Info("Loading environment from", "file", cli.Flags.AppConfigPath)
Expand All @@ -63,51 +39,45 @@ func init() {
// ------------------------------------------------------------------
// parse and load config from os exported
// ------------------------------------------------------------------
var err error
appConfig, err = appconfig.ParseConfig(ctx)
if err != nil {
slog.Error("error starting app, error parsing app config", "error", err)
os.Exit(appMainExitCode)
} // .if

// ------------------------------------------------------------------
// configure app custom logging
// ------------------------------------------------------------------
logInfo = append(logInfo, "pkg", "main")
logger = cli.AppLogger(appConfig).With(logInfo...)
sloger.SetDefaultLogger(logger)

explogger := cli.ExpAppLogger(appConfig).With(logInfo...)
slogerxexp.SetDefaultLogger(explogger)

}

func main() {
ctx, cancelFunc := context.WithCancel(context.Background())
var mainWaitGroup sync.WaitGroup

logger.Info("starting app")
appConfig, err := appconfig.ParseConfig(ctx)
if err != nil {
slog.Error("error starting app, error parsing app config", "error", err)
os.Exit(appMainExitCode)
} // .if

sloger.SetDefaultLogger(cli.AppLogger(appConfig))
slog.SetDefault(sloger.DefaultLogger)
slogerxexp.SetDefaultLogger(cli.ExpAppLogger(appConfig))
slog.Info("starting app")

// Pub Sub
event.MaxRetries = appConfig.EventMaxRetryCount
// initialize event reporter
err := cli.InitReporters(ctx, appConfig)
if err := cli.InitReporters(ctx, appConfig); err != nil {
slog.Error("error creating reporters", "error", err)
os.Exit(appMainExitCode)
}
defer reports.CloseAll()

event.InitFileReadyChannel()
defer event.CloseFileReadyChannel()

err = event.InitFileReadyPublisher(ctx, appConfig)
defer event.FileReadyPublisher.Close()
if err != nil {
logger.Error("error creating file ready publisher", "error", err)
if err := event.InitFileReadyPublisher(ctx, appConfig); err != nil {
slog.Error("error creating file ready publisher", "error", err)
os.Exit(appMainExitCode)
}
defer event.FileReadyPublisher.Close()

mainWaitGroup.Add(1)
subscriber, err := cli.NewEventSubscriber[*event.FileReady](ctx, appConfig)
if err != nil {
logger.Error("error subscribing to file ready", "error", err)
slog.Error("error subscribing to file ready", "error", err)
os.Exit(appMainExitCode)
}
defer subscriber.Close()
Expand All @@ -119,11 +89,11 @@ func main() {
// start serving the app
handler, err := cli.Serve(ctx, appConfig)
if err != nil {
logger.Error("error starting app, error initialize dex handler", "error", err)
slog.Error("error starting app, error initialize dex handler", "error", err)
os.Exit(appMainExitCode)
}

logger.Info("http handlers ready")
slog.Info("http handlers ready")
// ------------------------------------------------------------------
// Start http custom server
// ------------------------------------------------------------------
Expand All @@ -140,24 +110,26 @@ func main() {
go func() {
err := httpServer.ListenAndServe()
if err != nil && !errors.Is(err, http.ErrServerClosed) {
logger.Error("error starting app, error starting http server", "error", err, "port", appConfig.ServerPort)
slog.Error("error starting app, error starting http server", "error", err, "port", appConfig.ServerPort)
os.Exit(appMainExitCode)
} // .if
mainWaitGroup.Done()
}() // .go

logger.Info("started http server with tusd and dex handlers", "port", appConfig.ServerPort)
slog.Info("started http server with tusd and dex handlers", "port", appConfig.ServerPort)

mainWaitGroup.Add(1)
go func() {
defer mainWaitGroup.Done()
if err := ui.Start(appConfig.UIPort, appConfig.CsrfToken, appConfig.TusUIFileEndpointUrl, appConfig.TusUIInfoEndpointUrl); err != nil {
logger.Error("failed to start ui", "error", err)
os.Exit(appMainExitCode)
}
}()
if appConfig.UIPort != "" {
mainWaitGroup.Add(1)
go func() {
defer mainWaitGroup.Done()
if err := ui.Start(appConfig.UIPort, appConfig.CsrfToken, appConfig.TusUIFileEndpointUrl, appConfig.TusUIInfoEndpointUrl); err != nil {
slog.Error("failed to start ui", "error", err)
os.Exit(appMainExitCode)
}
}()

logger.Info("Started ui server", "port", appConfig.UIPort)
slog.Info("Started ui server", "port", appConfig.UIPort)
}

// ------------------------------------------------------------------
// Block for Exit, server above is on goroutine
Expand All @@ -176,5 +148,5 @@ func main() {

mainWaitGroup.Wait()

logger.Info("closing server by os signal", "port", appConfig.ServerPort)
slog.Info("closing server by os signal", "port", appConfig.ServerPort)
} // .main
Loading

0 comments on commit 03f3aae

Please sign in to comment.