From 8f015f76c35965780ee023b33385ad6471fc59e5 Mon Sep 17 00:00:00 2001 From: "Kevin S. Clarke" Date: Mon, 24 Feb 2025 15:32:16 -0500 Subject: [PATCH 1/7] [SERV-1212] Add OpenAPI spec, Echo handling --- .github/workflows/build.yml | 2 +- .github/workflows/nightly.yml | 12 ++ .github/workflows/release.yml | 16 ++ .gitignore | 3 + Dockerfile | 19 ++- Makefile | 45 ++++-- README.md | 109 +++++-------- api/api.go | 227 ++++++++++++++++++++++++++++ api/api_test.go | 26 ++++ csvutils/csv_io.go | 24 +++ go.mod | 24 ++- go.sum | 43 ++++++ integration/docker_setup_test.go | 105 +++++++++++++ integration/log_utils_test.go | 112 ++++++++++++++ integration/status_test.go | 62 ++++++++ integration/upload_csv_test.go | 82 ++++++++++ main.go | 150 ++++++++++++++---- main_tc_test.go | 99 ------------ main_test.go | 81 +++++++--- openapi.yml | 129 ++++++++++++++++ testdata/cct-collection.csv | 2 +- validation/checks/ark_check_test.go | 13 +- validation/checks/eol_check_test.go | 9 -- validation/config/logger.go | 33 ++++ validation/config/logger_test.go | 63 ++++++++ validation/config/profiles.go | 22 ++- validation/config/profiles_test.go | 8 - validation/engine.go | 64 ++++---- 28 files changed, 1289 insertions(+), 295 deletions(-) create mode 100644 api/api.go create mode 100644 api/api_test.go create mode 100644 integration/docker_setup_test.go create mode 100644 integration/log_utils_test.go create mode 100644 integration/status_test.go create mode 100644 integration/upload_csv_test.go delete mode 100644 main_tc_test.go create mode 100644 openapi.yml create mode 100644 validation/config/logger.go create mode 100644 validation/config/logger_test.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4bdbdf9..ba37a59 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,5 +50,5 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} # Build and test the application (also lints) - - name: Build Docker container + - name: Build project run: make all diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8db968e..d05107e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -21,6 +21,17 @@ jobs: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # Set up the Go environment + - name: Setup Go + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 + with: + go-version: "${{ env.GO_VERSION }}" + + - name: Install Go linter + run: | + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ + sh -s -- -b $(go env GOPATH)/bin ${{ env.GO_LINTER_VERSION }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0 @@ -42,3 +53,4 @@ jobs: tags: ${{ env.DOCKER_REGISTRY_ACCOUNT }}/${{ env.SERVICE_NAME }}:nightly build-args: | SERVICE_NAME=${{ env.SERVICE_NAME }} + LOG_LEVEL=info diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 000ee96..4daea82 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,6 +20,17 @@ jobs: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # Set up the Go environment + - name: Setup Go + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 + with: + go-version: "${{ env.GO_VERSION }}" + + - name: Install Go linter + run: | + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ + sh -s -- -b $(go env GOPATH)/bin ${{ env.GO_LINTER_VERSION }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0 @@ -32,6 +43,10 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + # Build and test the application (also lints) + - name: Build project + run: make all + - name: Build and push Docker image uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 # v6.14.0 with: @@ -41,3 +56,4 @@ jobs: tags: ${{ env.DOCKER_REGISTRY_ACCOUNT }}/${{ env.SERVICE_NAME }}:${{ github.ref_name }} build-args: | SERVICE_NAME=${{ env.SERVICE_NAME }} + LOG_LEVEL=info diff --git a/.gitignore b/.gitignore index b3b60a1..831b937 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ go.work # Build artifacts *-service + +# A profile configuration for the validation service +profiles.json diff --git a/Dockerfile b/Dockerfile index 78e9b76..47590c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ LABEL org.opencontainers.image.source="https://github.com/uclalibrary/${SERVICE_ LABEL org.opencontainers.image.description="UCLA Library's ${SERVICE_NAME} container" # Set the working directory inside the container -WORKDIR /app +WORKDIR /service # Copy the local package files to the container COPY . . @@ -31,21 +31,36 @@ RUN go build -o "/${SERVICE_NAME}" ## FROM alpine:3.21 +# Define the location of our application data directory +ARG DATA_DIR="/usr/local/data" + # Inherit SERVICE_NAME arg and set as ENV ARG SERVICE_NAME ENV SERVICE_NAME=${SERVICE_NAME} +# Inherit LOG_LEVEL arg and set as ENV +ARG LOG_LEVEL +ENV LOG_LEVEL=${LOG_LEVEL} + +# Set the location of the profiles config +ENV PROFILES_FILE="$DATA_DIR/profiles.json" + # Install curl to be used in container healthcheck RUN apk add --no-cache curl # Create a non-root user RUN addgroup -S "${SERVICE_NAME}" && adduser -S "${SERVICE_NAME}" -G "${SERVICE_NAME}" +# Create a directory for our profiles file +RUN mkdir -p "$DATA_DIR" + # Copy the file without --chown or --chmod (BuildKit not required) COPY --from=build "/${SERVICE_NAME}" "/sbin/${SERVICE_NAME}" +COPY "testdata/test_profiles.json" "$PROFILES_FILE" # Now, modify ownership and permissions in a separate RUN step RUN chown "${SERVICE_NAME}":"${SERVICE_NAME}" "/sbin/${SERVICE_NAME}" && chmod 0700 "/sbin/${SERVICE_NAME}" +RUN chown "${SERVICE_NAME}":"${SERVICE_NAME}" "$PROFILES_FILE" && chmod 0700 "$PROFILES_FILE" # Expose the port on which the application will run EXPOSE 8888 @@ -57,4 +72,4 @@ USER "${SERVICE_NAME}" ENTRYPOINT [ "sh", "-c", "exec /sbin/${SERVICE_NAME}" ] # Confirm the service started as expected -HEALTHCHECK CMD curl -f http://localhost:8888/ || exit 1 +HEALTHCHECK CMD curl -f http://localhost:8888/status || exit 1 diff --git a/Makefile b/Makefile index 17ff730..4a50196 100644 --- a/Makefile +++ b/Makefile @@ -1,30 +1,53 @@ -# Build variables +# Build and runtime variables SERVICE_NAME := validation-service LOG_LEVEL := info +PORT := 8888 # Do a full build of the project -all: lint build test docker-test +all: api lint build test docker-test + +# Lint the code +lint: + golangci-lint run + +# Generate Go code from the OpenAPI specification only when it has changed +api/api.go: openapi.yml + oapi-codegen -package api -generate types,server,spec -o api/api.go openapi.yml + +# This is an alias for the longer API generation Makefile target api/api.go +api: api/api.go # Build the Go project build: go build -o $(SERVICE_NAME) -# Run Go tests +# Run Go tests, excluding tests in the 'integration' directory test: go test -tags=unit ./... -v -args -log-level=$(LOG_LEVEL) -# Lint the code -lint: - golangci-lint run - # Build the Docker container (an optional debugging step) docker-build: - docker build . + docker build . --tag $(SERVICE_NAME) --build-arg SERVICE_NAME=$(SERVICE_NAME) + +# A convenience target to assist with running the Docker container outside of the build (optional) +docker-run: + docker run -p $(PORT):8888 --name $(SERVICE_NAME) -d $(shell docker image ls -q --filter=reference=$(SERVICE_NAME)) -# Run tests inside the Docker container +docker-logs: + docker logs -f $(shell docker ps --filter "name=$(SERVICE_NAME)" --format "{{.ID}}") + +# A convenience target to assist with stopping the Docker container outside of the build (optional) +docker-stop: + docker rm -f $(shell docker ps --filter "name=$(SERVICE_NAME)" --format "{{.ID}}") + +# Run tests inside the Docker container (does not require docker-build, that's just for debugging) docker-test: - go test -tags=functional . -v -args -service-name=$(SERVICE_NAME) + go test -tags=integration ./integration -v -args -service-name=$(SERVICE_NAME) -log-level=$(LOG_LEVEL) # Clean up all artifacts of the build clean: - rm -rf $(SERVICE_NAME) + rm -rf $(SERVICE_NAME) api/api.go + +# Run the validation service locally +run: api build + PROFILES_FILE="testdata/test_profiles.json" ./$(SERVICE_NAME) diff --git a/README.md b/README.md index cf4daee..60a5906 100644 --- a/README.md +++ b/README.md @@ -7,115 +7,86 @@ Note: This project is in its infancy and is not ready for general use. ## Getting Started -There are multiple ways to build the project. You are free to use whichever you prefer. A series of manual steps is -provided below, but there are also more concise build processes available for [Make](#building-and-running-with-make) -and [ACT](#building-and-running-with-act). +The recommended way to build and start using the project is to use the project's [Makefile](Makefile). This will +require installing GNU Make. How you do this will depend on which OS (Linux, Mac, Windows) you are using. Consult +your system's documentation or package system for more details. ### Prerequisites +* The [GNU Make](https://www.gnu.org/software/make/) tool to run the project's Makefile * A [GoLang](https://go.dev/doc/install) build environment * A functional [Docker](https://docs.docker.com/get-started/get-docker/) installation * The [golangci-lint](https://github.com/golangci/golangci-lint) linter for checking code style conformance Optionally, if you want to test (or build using) the project's GitHub Actions: - * [ACT](https://github.com/nektos/act): A local GitHub Action runner that will also build and test the project -Additionally, Make can be used as a simpler build tool for the project. It should be installed through your OS' -standard packaging system. - -### Building the Application - -To build the project, type: - -`go build -o validation-service` - -To run the service locally, type: - -`./validation-service` - -Typing `Ctrl-C` will stop the service. - -### Running the Test Suite - -There are unit and functional tests (the latter of which require a working Docker installation). - -To run the unit tests, type: - -`go test -tags=unit ./... -v` - -To run the functional tests, type: - -`go test -tags=functional ./... -v -args -service-name=validation-service` - -Note that the functional tests will spin up a Docker container and run tests against that. - -### Running the Linter - -To run the project's linter, type: +## Building and Running with Make -`golangci-lint run` +The project's [Makefile](Makefile) provides a convenient way to build, test, lint, and manage Docker containers for the +project. This is the method we recommend. -### Spinning up the Docker Container (Independent of the Tests) +The TL;DR is that running `make all` will perform all the project's required build and testing steps. Individual steps +(listed below) are also available, though, for a more targeted approach. -To build the Docker image, type: +### Commands -`docker build -t validation-service --build-arg SERVICE_NAME="validation-service" .` +To generate Go code from the project's OpenAPI specification: -To run the newly built Docker image, type: + make api -`docker run -d -p 8888:8888 --name validation-service validation-service` +To build the project: -Once the container is running, you can access the service at: + make build -`http://localhost:8888/` +To run all the unit tests: -To stop the service and remove the Docker container, type: + make test -`docker rm -f validation-service` +To run the integration tests, which includes building the Docker container: -### Cleaning up the Project's Build Artifacts + make docker-test -To clean up the project's build artifacts, type: +To run the linter: -`rm -rf validation-service` + make lint -To simplify your processes, though, we recommend that you use Make, which has a simpler command line interface. +To clean up the project's build artifacts, run: -## Building and Running with Make + make clean -The project's [Makefile](Makefile) provides another convenient way to build, test, lint, and manage Docker containers -for the project. This is the method we recommend. +Note: If you want to change the values defined in the Makefile (echo.g., the `LOG_LEVEL`), these can be supplied to the +`make` command: -The TL;DR is that running `make all` will perform all the project's required build and testing steps. Individual steps -(listed below) are also available, though, for a more targeted approach. + make test LOG_LEVEL=debug -### Commands +To run the validation service, locally, for testing purposes: -To build the Go project: + make run - make build +### Working with Docker -To run all the unit tests: +One can also run Docker locally, for more hands-on testing, from the Makefile. Unlike the tests, which will not leave +Docker containers in your local Docker repo, these targets build and run the container using the Docker command line. - make test +To build a Docker container to your local Docker repo: -To run the linter: + make docker-build - make lint +To run a Docker container that's already been built: -To run the functional tests, which includes building the Docker container: + make docker-run - make docker-test +To see the logs, in real time, from that Docker container: -To clean up the project's build artifacts, run: + make docker-logs - make clean +To stop the Docker container when you are done with it: -Note: If you want to change the values defined in the Makefile (e.g., the `LOG_LEVEL`), these can be supplied to the -`make` command: + make docker-stop - make test LOG_LEVEL=debug +Note: None of the Docker specific Makefile targets (except `docker-test`) are required to build or test the project. +They are just additional conveniences for developers. ## Building and Running with ACT diff --git a/api/api.go b/api/api.go new file mode 100644 index 0000000..d4d9b16 --- /dev/null +++ b/api/api.go @@ -0,0 +1,227 @@ +// Package api provides primitives to interact with the openapi HTTP API. +// +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.4.1 DO NOT EDIT. +package api + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "fmt" + "net/http" + "net/url" + "path" + "strings" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/labstack/echo/v4" + "github.com/oapi-codegen/runtime" + openapi_types "github.com/oapi-codegen/runtime/types" +) + +// Status A JSON document representing the service's runtime status. It's intentionally brief, for now. +type Status struct { + FS string `json:"fs"` + S3 string `json:"s3"` + Service string `json:"service"` +} + +// JobIDParam defines model for JobIDParam. +type JobIDParam = string + +// StatusCreated A JSON document representing the service's runtime status. It's intentionally brief, for now. +type StatusCreated = Status + +// StatusOK A JSON document representing the service's runtime status. It's intentionally brief, for now. +type StatusOK = Status + +// UploadCSVMultipartBody defines parameters for UploadCSV. +type UploadCSVMultipartBody struct { + // CsvFile The CSV file to be uploaded + CsvFile openapi_types.File `json:"csvFile"` + + // Profile The name of the profile the validation process should use + Profile string `json:"profile"` +} + +// UploadCSVMultipartRequestBody defines body for UploadCSV for multipart/form-data ContentType. +type UploadCSVMultipartRequestBody UploadCSVMultipartBody + +// ServerInterface represents all server handlers. +type ServerInterface interface { + // Gets the validation service's current status + // (GET /status) + GetStatus(ctx echo.Context) error + + // (GET /status/{jobID}) + GetJobStatus(ctx echo.Context, jobID JobIDParam) error + // Start a new validation process + // (POST /upload/csv) + UploadCSV(ctx echo.Context) error +} + +// ServerInterfaceWrapper converts echo contexts to parameters. +type ServerInterfaceWrapper struct { + Handler ServerInterface +} + +// GetStatus converts echo context to params. +func (w *ServerInterfaceWrapper) GetStatus(ctx echo.Context) error { + var err error + + // Invoke the callback with all the unmarshaled arguments + err = w.Handler.GetStatus(ctx) + return err +} + +// GetJobStatus converts echo context to params. +func (w *ServerInterfaceWrapper) GetJobStatus(ctx echo.Context) error { + var err error + // ------------- Path parameter "jobID" ------------- + var jobID JobIDParam + + err = runtime.BindStyledParameterWithOptions("simple", "jobID", ctx.Param("jobID"), &jobID, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter jobID: %s", err)) + } + + // Invoke the callback with all the unmarshaled arguments + err = w.Handler.GetJobStatus(ctx, jobID) + return err +} + +// UploadCSV converts echo context to params. +func (w *ServerInterfaceWrapper) UploadCSV(ctx echo.Context) error { + var err error + + // Invoke the callback with all the unmarshaled arguments + err = w.Handler.UploadCSV(ctx) + return err +} + +// This is a simple interface which specifies echo.Route addition functions which +// are present on both echo.Echo and echo.Group, since we want to allow using +// either of them for path registration +type EchoRouter interface { + CONNECT(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route + DELETE(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route + GET(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route + HEAD(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route + OPTIONS(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route + PATCH(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route + POST(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route + PUT(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route + TRACE(path string, h echo.HandlerFunc, m ...echo.MiddlewareFunc) *echo.Route +} + +// RegisterHandlers adds each server route to the EchoRouter. +func RegisterHandlers(router EchoRouter, si ServerInterface) { + RegisterHandlersWithBaseURL(router, si, "") +} + +// Registers handlers, and prepends BaseURL to the paths, so that the paths +// can be served under a prefix. +func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL string) { + + wrapper := ServerInterfaceWrapper{ + Handler: si, + } + + router.GET(baseURL+"/status", wrapper.GetStatus) + router.GET(baseURL+"/status/:jobID", wrapper.GetJobStatus) + router.POST(baseURL+"/upload/csv", wrapper.UploadCSV) + +} + +// Base64 encoded, gzipped, json marshaled Swagger object +var swaggerSpec = []string{ + + "H4sIAAAAAAAC/7xWTW/jRgz9K8S0QC6K5e2mF9/SbFM4RXcXdZrLdg8jDW1NIs2oQ8qOEfi/Fxx9+CNK", + "E3SBvdnSzOPj4yOpJ5X7qvYOHZOaPalaB10hY4j/bnw2//BZHsk/g5QHW7P1Ts3UpYP5B1j6AFwgrHVp", + "jZZXcO8zlSgrZ2rNhUqU0xWqmboXNJWogP80NqBRMw4NJoryAistEXhby0HiYN1K7XY7OUy1d4SRz9wx", + "BqfLBYY1hl9D8EEe594xOo4I+MhpXWoJ/3SAjI+6qksBvy0QiDU3BEIEiWGpbYkGMsx1QxjzodJysQX2", + "ayQw1sBqG1AlYwyPZbktMCBsNIF2YDu+QJEwYGS8S9RHz9e+ceb/p9BxRwMByTchRzj7Y/tn9/sMct+U", + "BpxnyBCWEuuN7MeQJR2B0lmJwH4PuUvUImp5FVCzlPQoFV3Xpc2jLdJ78icJ/RhwqWbqh3RvwbR9S2mL", + "OkbxEnpLgHUmortVrNkI8UITZIgO8o7fQPjT79+XKxeaISA3wRFouFl8+gg+u8ecYWO56C1p3dKHKrJQ", + "gtfFEApdmOeN2IIZnzcVOglSByR0gy5iPpvjGUFoHNuq9/8E5nxG0aROsHRZbiELFpdJ7GvnNxOVqDr4", + "GgPbtgWXdOxF//DMV4l6PF/5867rrxciOr1/9Zqcapm+fnR3OEW+DPdinEQ4fh2utCK31RF1xwQ8mF4d", + "VFuvvMD8geBqcQdLWyJBQ2jAOvBNAGNXlnUJGx8elmWrFVuOnO/2gIsO8PLzXCVqjYHaqNPJdPJOkvY1", + "Ol1bNVPvJ9PJhSiuuYgypzTUfIX8nPltYQnQmdpb95/uOrAV6Mw33Bqj9ZxfHtpE0pCCx8Nzo2bqN+TO", + "eyfj+Kfp9KXeGM6lQ7vtEvXzWy6MDfnYCk1V6bBtCdHp1tmbPG9CkD6gvi2TXsf0Ka6g3Yt63lncnIyS", + "e5+dUY81osyNzwZxDpfnl/E090fSg+W6+/qtyl5ML16/cLx1vq0eUdamLr02aU7r+O3g6VWPEuvAYlGH", + "m8Py1cHnSNJgw9RqZCCjkVfSfKCdiZ3YBv3bPavGX/HF1eKu+8ZA4l+82Z7M+aop2dY6cCotcW406+NR", + "fzzvclpf2xLH8sJhMHQ7sWWGsmnbdlMzlVknph0Zdl1e48gyPPvG7AU4cXwvGRVx1TeEr87JPpl98PE5", + "efyFtntmzndvNefVfu1+d4ceToyF2O5F10U7/xsAAP//oXcG8woLAAA=", +} + +// GetSwagger returns the content of the embedded swagger specification file +// or error if failed to decode +func decodeSpec() ([]byte, error) { + zipped, err := base64.StdEncoding.DecodeString(strings.Join(swaggerSpec, "")) + if err != nil { + return nil, fmt.Errorf("error base64 decoding spec: %w", err) + } + zr, err := gzip.NewReader(bytes.NewReader(zipped)) + if err != nil { + return nil, fmt.Errorf("error decompressing spec: %w", err) + } + var buf bytes.Buffer + _, err = buf.ReadFrom(zr) + if err != nil { + return nil, fmt.Errorf("error decompressing spec: %w", err) + } + + return buf.Bytes(), nil +} + +var rawSpec = decodeSpecCached() + +// a naive cached of a decoded swagger spec +func decodeSpecCached() func() ([]byte, error) { + data, err := decodeSpec() + return func() ([]byte, error) { + return data, err + } +} + +// Constructs a synthetic filesystem for resolving external references when loading openapi specifications. +func PathToRawSpec(pathToFile string) map[string]func() ([]byte, error) { + res := make(map[string]func() ([]byte, error)) + if len(pathToFile) > 0 { + res[pathToFile] = rawSpec + } + + return res +} + +// GetSwagger returns the Swagger specification corresponding to the generated code +// in this file. The external references of Swagger specification are resolved. +// The logic of resolving external references is tightly connected to "import-mapping" feature. +// Externally referenced files must be embedded in the corresponding golang packages. +// Urls can be supported but this task was out of the scope. +func GetSwagger() (swagger *openapi3.T, err error) { + resolvePath := PathToRawSpec("") + + loader := openapi3.NewLoader() + loader.IsExternalRefsAllowed = true + loader.ReadFromURIFunc = func(loader *openapi3.Loader, url *url.URL) ([]byte, error) { + pathToFile := url.String() + pathToFile = path.Clean(pathToFile) + getSpec, ok := resolvePath[pathToFile] + if !ok { + err1 := fmt.Errorf("path not found: %s", pathToFile) + return nil, err1 + } + return getSpec() + } + var specData []byte + specData, err = rawSpec() + if err != nil { + return + } + swagger, err = loader.LoadFromData(specData) + if err != nil { + return + } + return +} diff --git a/api/api_test.go b/api/api_test.go new file mode 100644 index 0000000..50df293 --- /dev/null +++ b/api/api_test.go @@ -0,0 +1,26 @@ +//go:build unit + +package api + +import ( + "flag" + "fmt" + "github.com/UCLALibrary/validation-service/testflags" + "testing" +) + +// TestAPI loads the flags for the tests in the 'api' package. +func TestAPI(t *testing.T) { + flag.Parse() + fmt.Printf("%s's log level: %s\n", t.Name(), *testflags.LogLevel) +} + +// TestForTheSakeOfSuccess is a simple brain-dead test that just prevents the build from failing for lack of tests. +// +// We don't actually test anything in the 'api' package. It's all code autogenerated by the oapi-codegen tool. +func TestForTheSakeOfSuccess(t *testing.T) { + _, err := GetSwagger() + if err != nil { + t.Errorf("GetSwagger() error = %v", err) + } +} diff --git a/csvutils/csv_io.go b/csvutils/csv_io.go index b402f5c..992cd62 100644 --- a/csvutils/csv_io.go +++ b/csvutils/csv_io.go @@ -7,9 +7,33 @@ import ( "encoding/csv" "fmt" "go.uber.org/zap" + "mime/multipart" "os" ) +// ReadUpload reads the CSV file from the supplied FileHeader and returns a string matrix. +func ReadUpload(fileHeader *multipart.FileHeader, logger *zap.Logger) ([][]string, error) { + file, err := fileHeader.Open() + if err != nil { + logger.Error("Failed to open uploaded file", zap.Error(err)) + } + defer func() { + if err := file.Close(); err != nil { + logger.Error("failed to close file", zap.Error(err)) + } + }() + + // Create a new CSV reader from the opened CSV file + reader := csv.NewReader(file) + + // Read all records from the CSV reader + if csvData, err := reader.ReadAll(); err != nil || len(csvData) < 1 { + return nil, fmt.Errorf("failed to parse file '%s': %w", fileHeader.Filename, err) + } else { + return csvData, nil + } +} + // ReadFile reads the CSV file at the supplied file path and returns a string matrix. func ReadFile(filePath string, logger *zap.Logger) ([][]string, error) { file, err := os.Open(filePath) diff --git a/go.mod b/go.mod index 3c25cd7..e4ab39c 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( dario.cat/mergo v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/containerd/containerd v1.7.18 // indirect github.com/containerd/log v0.1.0 // indirect @@ -23,26 +24,38 @@ require ( github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/getkin/kin-openapi v0.124.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.20.2 // indirect + github.com/go-openapi/swag v0.22.8 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/mux v1.8.1 // indirect + github.com/invopop/yaml v0.2.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/karagenc/zap4echo v0.1.1 // indirect github.com/klauspost/compress v1.17.7 // indirect github.com/kr/text v0.2.0 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/sys/user v0.1.0 // indirect github.com/moby/term v0.5.0 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/oapi-codegen/echo-middleware v1.0.2 // indirect + github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect @@ -60,10 +73,11 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/crypto v0.33.0 // indirect + golang.org/x/net v0.35.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect + golang.org/x/time v0.8.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 279eeda..403dde6 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,10 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= @@ -32,6 +36,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M= +github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -40,6 +46,10 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= +github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= +github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw= +github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -48,8 +58,17 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= +github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/karagenc/zap4echo v0.1.1 h1:sTsTP/tzQ8/rAATngapjG4QyzPdFIArlFMJu6Nky+fQ= +github.com/karagenc/zap4echo v0.1.1/go.mod h1:/oM1dL1YvoUxZvpVaP8jwzUqRVtE/xorr6LdmV4gPy8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= @@ -67,8 +86,12 @@ github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIg github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -82,12 +105,20 @@ github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/oapi-codegen/echo-middleware v1.0.2 h1:oNBqiE7jd/9bfGNk/bpbX2nqWrtPc+LL4Boya8Wl81U= +github.com/oapi-codegen/echo-middleware v1.0.2/go.mod h1:5J6MFcGqrpWLXpbKGZtRPZViLIHyyyUHlkqg6dT2R4E= +github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= +github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= +github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -105,11 +136,13 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -158,6 +191,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -166,6 +201,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -184,12 +221,17 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -212,6 +254,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= diff --git a/integration/docker_setup_test.go b/integration/docker_setup_test.go new file mode 100644 index 0000000..7bd4c6a --- /dev/null +++ b/integration/docker_setup_test.go @@ -0,0 +1,105 @@ +//go:build integration + +// Package integration holds the project's integration tests. +// +// This file sets up the Docker container for integration testing. +package integration + +import ( + docker "context" + "flag" + "fmt" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "go.uber.org/zap" + "log" + "os" + "testing" +) + +// Define our test container's build arguments +var serviceName string +var logLevel string + +// The URL to which to submit test HTTP requests +var testServerURL string + +// A reference to our Docker container +var container testcontainers.Container + +// Initialize our service name flag. +func init() { + flag.StringVar(&serviceName, "service-name", "service", "Name of service being tested") + flag.StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)") +} + +// TestMain spins up a Docker container with our validation service to run tests against. +func TestMain(m *testing.M) { + flag.Parse() + + // Creates a logger for our tests + logger, _ = getLogger() + //noinspection GoUnhandledErrorResult + defer logger.Sync() + + // Get the Docker context + context := docker.Background() + + // Define the container request + request := testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Context: "..", + Dockerfile: "Dockerfile", + BuildArgs: map[string]*string{ + "SERVICE_NAME": &serviceName, + "LOG_LEVEL": &logLevel, + }, + }, + ExposedPorts: []string{"8888/tcp"}, + WaitingFor: wait.ForHTTP("/status").WithPort("8888/tcp"), + LogConsumerCfg: &testcontainers.LogConsumerConfig{ + Consumers: []testcontainers.LogConsumer{&DockerLogConsumer{}}, + }, + } + + // Disable unnecessary output logging from the test containers + testcontainers.Logger = &FilteredLogger{ + original: log.New(log.Writer(), "", log.LstdFlags), + } + + // Start the container + container, containerErr := testcontainers.GenericContainer(context, testcontainers.GenericContainerRequest{ + ContainerRequest: request, + Started: true, + }) + if containerErr != nil { + logger.Fatal("Failed to start Docker container", zap.Error(containerErr)) + } + //noinspection GoUnhandledErrorResult + defer container.Terminate(context) + + // Get the mapped host and port + host, hostErr := container.Host(context) + if hostErr != nil { + logger.Fatal("Failed to get container host", zap.Error(hostErr)) + } + + port, portErr := container.MappedPort(context, "8888") + if portErr != nil { + logger.Fatal("Failed to get container port", zap.Error(portErr)) + } + + // Store the connect information for reuse in tests + testServerURL = fmt.Sprintf("http://%s:%d", host, port.Int()) + "%s" + + // Run tests + code := m.Run() + + // Cleanup: Stop the container after all tests + exitErr := container.Terminate(context) + if exitErr != nil { + logger.Fatal("Failed to terminate Docker container", zap.Error(exitErr)) + } + + os.Exit(code) +} diff --git a/integration/log_utils_test.go b/integration/log_utils_test.go new file mode 100644 index 0000000..e7685b8 --- /dev/null +++ b/integration/log_utils_test.go @@ -0,0 +1,112 @@ +//go:build integration + +// Package integration holds the project's integration tests. +// +// This file contains utilities for working with test loggers. +package integration + +import ( + "fmt" + "github.com/testcontainers/testcontainers-go" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "log" + "strings" + "sync" + "time" +) + +// A logger to log our Docker container's logs +var logger *zap.Logger + +// DockerLogConsumer implements the LogConsumer interface to capture container logs. +type DockerLogConsumer struct { + mutex sync.Mutex +} + +// FilteredLogger filters specific TestContainer messages. +type FilteredLogger struct { + original *log.Logger +} + +// getLogger builds a logger for use in relaying Docker container logs. +func getLogger() (*zap.Logger, error) { + config := zap.NewDevelopmentConfig() + + config.EncoderConfig.TimeKey = "" // Remove Time Field + config.EncoderConfig.LevelKey = "" // Remove Level Field + config.EncoderConfig.CallerKey = "" // Remove Caller Field + config.EncoderConfig.MessageKey = "message" + + // Simplify the label of the logging output + config.EncoderConfig.EncodeTime = func(time time.Time, enc zapcore.PrimitiveArrayEncoder) { + enc.AppendString("[DOCKER]") + } + + // Get rid of line breaks in the output + config.EncoderConfig.ConsoleSeparator = "" + + // Ensure logs are printed as plain text + config.Encoding = "console" + config.OutputPaths = []string{"stdout"} + + // Build and return the logger + logger, _ := config.Build() + //noinspection GoUnhandledErrorResult + defer logger.Sync() + + return logger, nil +} + +// shouldFilterLog determines if the log message should be filtered or not. +func shouldFilterLog(message string) bool { + return strings.Contains(message, "⏳") || strings.Contains(message, "✅") +} + +// Accept relays the log entries from our Docker container to our test logger. +func (consumer *DockerLogConsumer) Accept(log testcontainers.Log) { + consumer.mutex.Lock() + defer consumer.mutex.Unlock() + + // Output our container's logs to our test logger + logger.Debug(string(log.Content)) +} + +// Print prints the log messages we're not filtering. +func (logger *FilteredLogger) Print(log ...interface{}) { + message := fmt.Sprint(log...) + + // Suppress logs that contain don't provide useful information + if shouldFilterLog(message) { + return + } + + // Allow other logs to pass through + logger.original.Print(log...) +} + +// Println prints the log messages we're not filtering. +func (logger *FilteredLogger) Println(log ...interface{}) { + message := strings.TrimSpace(strings.Join(strings.Fields(strings.Trim(fmt.Sprint(log...), "[]")), " ")) + + // Suppress logs that contain don't provide useful information + if shouldFilterLog(message) { + return + } + + // Allow all other logs to pass through + logger.original.Println(log...) +} + +// Printf prints the log messages we're not filtering. +func (logger *FilteredLogger) Printf(format string, log ...interface{}) { + message := fmt.Sprintf(format, log...) + + // Suppress logs that contain don't provide useful information + if shouldFilterLog(message) { + return + } + + // Allow all other logs to pass through + logger.original.Printf(format, log...) +} diff --git a/integration/status_test.go b/integration/status_test.go new file mode 100644 index 0000000..a6b55c1 --- /dev/null +++ b/integration/status_test.go @@ -0,0 +1,62 @@ +//go:build integration + +// Package integration holds the project's integration tests. +// +// This file contains tests of the service's `getStatus` endpoint. +package integration + +import ( + "bytes" + "fmt" + "github.com/stretchr/testify/assert" + "io" + "net/http" + "testing" + "time" +) + +// TestStatusGet tests the status endpoint to confirm that the server is responding as expected to status queries. +func TestStatusGet(t *testing.T) { + // Set up client for making requests to the containerized app + client := &http.Client{ + Timeout: 5 * time.Second, + } + + // Make a GET request to the containerized app and check the response + response, err := client.Get(fmt.Sprintf(testServerURL, "/status")) + if err != nil { + t.Fatal(err) + } + //noinspection GoUnhandledErrorResult + defer response.Body.Close() + + // Read the body into a variable to compare + body, readErr := io.ReadAll(response.Body) + if readErr != nil { + t.Fatalf("Error reading response: %v", readErr) + } + + assert.Equal(t, http.StatusOK, response.StatusCode) + assert.Equal(t, "{\"fs\":\"ok\",\"s3\":\"ok\",\"service\":\"ok\"}\n", string(body)) +} + +// TestStatusPost tests the status endpoint to confirm that the server doesn't respond to POST submissions. +func TestStatusPost(t *testing.T) { + // Set up client for making requests to the containerized app + client := &http.Client{ + Timeout: 5 * time.Second, + } + + // Make a POST request and confirm that POST is not supported + requestBody := []byte(`{"key": "value"}`) + + response, err := client.Post(fmt.Sprintf(testServerURL, "/status"), "application/json", + bytes.NewBuffer(requestBody)) + if err != nil { + t.Fatal(err) + } + //noinspection GoUnhandledErrorResult + defer response.Body.Close() + + assert.Equal(t, http.StatusNotFound, response.StatusCode) +} diff --git a/integration/upload_csv_test.go b/integration/upload_csv_test.go new file mode 100644 index 0000000..640a564 --- /dev/null +++ b/integration/upload_csv_test.go @@ -0,0 +1,82 @@ +//go:build integration + +// Package integration holds the project's integration tests. +// +// This file contains tests of the service's `uploadCSV` endpoint. +package integration + +import ( + "bytes" + "fmt" + "github.com/stretchr/testify/assert" + "io" + "mime/multipart" + "net/http" + "os" + "testing" +) + +// TestUploadCSV tests the uploadCSV endpoint with a basic CSV. +func TestUploadCSV(t *testing.T) { + csvFile := "../testdata/cct-works-simple.csv" + + // Open the test CSV file + file, openErr := os.Open(csvFile) + if openErr != nil { + t.Fatalf("Error opening file: %v", openErr) + } + //noinspection GoUnhandledErrorResult + defer file.Close() + + // Create a buffer to store the multipart form data + upload := &bytes.Buffer{} + writer := multipart.NewWriter(upload) + + // Create a form file field named 'csvFile' + part, formErr := writer.CreateFormFile("csvFile", csvFile) + if formErr != nil { + t.Fatalf("Error creating form file: %v", formErr) + } + + // Copy the CSV file content into the multipart form + if _, err := io.Copy(part, file); err != nil { + t.Fatalf("Error copying file content: %v", err) + } + + // Add the 'profile' field, too + _ = writer.WriteField("profile", "test") + + // Close the multipart writer to finalize the form + if err := writer.Close(); err != nil { + t.Fatalf("Error closing writer: %v", err) + } + + // Create an HTTP request to our CSV upload endpoint + url := fmt.Sprintf(testServerURL, "/upload/csv") + request, reqErr := http.NewRequest("POST", url, upload) + if reqErr != nil { + t.Fatalf("Error creating request: %v", reqErr) + } + + // Set the Content-Type header to multipart/form-data with the correct boundary + request.Header.Set("Content-Type", writer.FormDataContentType()) + + // Initialize an HTTP client and send the request + client := &http.Client{} + response, postErr := client.Do(request) + if postErr != nil { + t.Fatalf("Error sending request: %v", postErr) + } + //noinspection GoUnhandledErrorResult + defer response.Body.Close() + + // Read the body into a variable to compare + body, readErr := io.ReadAll(response.Body) + if readErr != nil { + t.Fatalf("Error reading response: %v", readErr) + } + + // Placeholder body check + assert.Equal(t, "{\"fs\":\"ok\",\"s3\":\"ok\",\"service\":\"created\"}\n", string(body)) + assert.Equal(t, http.StatusCreated, response.StatusCode) +} diff --git a/main.go b/main.go index db9c161..7965bb2 100644 --- a/main.go +++ b/main.go @@ -1,45 +1,141 @@ package main import ( - "net/http" - "strconv" - + "errors" + "fmt" + "github.com/UCLALibrary/validation-service/api" + "github.com/UCLALibrary/validation-service/csvutils" + "github.com/UCLALibrary/validation-service/validation" + "github.com/UCLALibrary/validation-service/validation/config" "github.com/labstack/echo/v4" + middleware "github.com/oapi-codegen/echo-middleware" + "go.uber.org/zap" + "log" + "net/http" ) +// Port is the default port for our server const Port = 8888 -// App represents your application. -type App struct { - Router *echo.Echo +// Service implements the generated API validation interface +type Service struct { + Engine *validation.Engine } -func main() { - app := NewApp() - app.Routes() - err := app.Router.Start(":" + strconv.Itoa(Port)) - - if err != nil { - panic(err) - } +// GetStatus handles the GET /status request +func (service *Service) GetStatus(context echo.Context) error { + // A placeholder response + return context.JSON(http.StatusOK, api.Status{ + Service: "ok", + S3: "ok", + FS: "ok", + }) } -// NewApp initializes a new instance of your application. -func NewApp() *App { - e := echo.New() - return &App{Router: e} +// GetJobStatus handles the GET /status/{jobID} request +func (service *Service) GetJobStatus(context echo.Context, jobID string) error { + // A placeholder response; structure will change + return context.JSON(http.StatusOK, api.Status{ + Service: "completed", + S3: "ok", + FS: "ok", + }) } -// Routes configure the server's endpoints. -func (app *App) Routes() { - app.Router.GET("/", helloWorld) - app.Router.RouteNotFound("/*", NotFoundHandler) -} +// UploadCSV handles the /upload/csv POST request +func (service *Service) UploadCSV(context echo.Context) error { + engine := service.Engine + logger := engine.GetLogger() -func helloWorld(c echo.Context) error { - return c.String(http.StatusOK, "hello world") + // Get the CSV file upload and profile + profile := context.FormValue("profile") + file, fileErr := context.FormFile("csvFile") + if fileErr != nil { + return context.JSON(http.StatusBadRequest, map[string]string{"error": "A CSV file must be uploaded"}) + } + + logger.Debug("Received uploaded CSV file", + zap.String("csvFile", file.Filename), + zap.String("profile", profile)) + + // Parse the CSV data + csvData, readErr := csvutils.ReadUpload(file, logger) + if readErr != nil { + return context.JSON(http.StatusBadRequest, map[string]string{"error": "Uploaded CSV file could not be parsed"}) + } + + if err := engine.Validate(profile, csvData); err != nil { + // Handle if there was a validation error + return context.JSON(http.StatusCreated, api.Status{ + Service: fmt.Sprintf("error: %v", err), + S3: "ok", + FS: "ok", + }) + } + + // Handle if there were no validation errors + return context.JSON(http.StatusCreated, api.Status{ + Service: "created", + S3: "ok", + FS: "ok", + }) } -func NotFoundHandler(c echo.Context) error { - return echo.NewHTTPError(http.StatusNotFound, "This is not yet supported") +// Main function starts our Echo server +func main() { + // Create a new validation engine + engine, err := validation.NewEngine() + if err != nil { + log.Fatal(err) + } + + // Create the validation service + service := &Service{ + Engine: engine, + } + + // Create a new validation application and configure its logger + echoApp := echo.New() + echoApp.Use(config.ZapLoggerMiddleware(engine.GetLogger())) + + // Hide the Echo startup messages + echoApp.HideBanner = true + echoApp.HidePort = true + + // Handle requests with and without a trailing slash + echoApp.Pre(func(next echo.HandlerFunc) echo.HandlerFunc { + return func(char echo.Context) error { + path := char.Request().URL.Path + + if path != "/" && path[len(path)-1] == '/' { + // Remove trailing slash as our canonical form + char.Request().URL.Path = path[:len(path)-1] + } + + return next(char) + } + }) + + // Load the OpenAPI spec for request validation + swagger, swaggerErr := api.GetSwagger() + if swaggerErr != nil { + log.Fatalf("Failed to load OpenAPI spec: %v", swaggerErr) + } + + // Register the Echo/OpenAPI validation middleware + echoApp.Use(middleware.OapiRequestValidator(swagger)) + + // Register request handlers + api.RegisterHandlers(echoApp, service) + + // Configure the validation echoApp + server := &http.Server{ + Addr: fmt.Sprintf(":%d", Port), + Handler: echoApp, + } + + // Start the validation echoApp + if err := echoApp.StartServer(server); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("Server failed: %v", err) + } } diff --git a/main_tc_test.go b/main_tc_test.go deleted file mode 100644 index c890efc..0000000 --- a/main_tc_test.go +++ /dev/null @@ -1,99 +0,0 @@ -//go:build functional - -package main - -import ( - "bytes" - "context" - "flag" - "fmt" - "net/http" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -// Configure our service name flag -var serviceName string - -// Initialize our service name flag -func init() { - flag.StringVar(&serviceName, "service-name", "service", "Name of service being tested") -} - -// TestApp spins up a Docker container with the application and runs simple tests against its Web API -func TestApp(t *testing.T) { - flag.Parse() - - // Define a Docker context - ctx := context.Background() - - // Define the container request - req := testcontainers.ContainerRequest{ - FromDockerfile: testcontainers.FromDockerfile{ - Context: ".", - Dockerfile: "Dockerfile", - BuildArgs: map[string]*string{ - "SERVICE_NAME": &serviceName, - }, - }, - ExposedPorts: []string{"8888/tcp"}, - WaitingFor: wait.ForHTTP("/").WithPort("8888/tcp"), - } - - // Start the container - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - - if err != nil { - t.Fatal(err) - } - - defer func() { - if err := container.Terminate(ctx); err != nil { - fmt.Printf("Error terminating container: %v\n", err) - } - }() - - // Get the host and port for the running container - host, err := container.Host(ctx) - if err != nil { - t.Fatal(err) - } - - port, err := container.MappedPort(ctx, "8888/tcp") - if err != nil { - t.Fatal(err) - } - - // Set up client for making requests to the containerized app - client := &http.Client{ - Timeout: 5 * time.Second, - } - - // Make a GET request to the containerized app and check the response - resp, err := client.Get(fmt.Sprintf("http://%s:%d/", host, port.Int())) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - assert.Equal(t, http.StatusOK, resp.StatusCode) - - // Make a POST request and confirm that POST is not currently supported - requestBody := []byte(`{"key": "value"}`) - - resp, err = client.Post(fmt.Sprintf("http://%s:%d/", host, port.Int()), "application/json", - bytes.NewBuffer(requestBody)) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - assert.Equal(t, http.StatusNotFound, resp.StatusCode) -} diff --git a/main_test.go b/main_test.go index 65c86ab..921273a 100644 --- a/main_test.go +++ b/main_test.go @@ -5,7 +5,12 @@ package main import ( "flag" "fmt" + "github.com/UCLALibrary/validation-service/api" "github.com/UCLALibrary/validation-service/testflags" + "github.com/UCLALibrary/validation-service/validation" + "github.com/UCLALibrary/validation-service/validation/config" + "github.com/labstack/echo/v4" + "github.com/stretchr/testify/require" "net/http" "net/http/httptest" "os" @@ -21,31 +26,67 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -// TestHelloWorld is a very simple initial test for the validation service application. -func TestHelloWorld(t *testing.T) { - app := NewApp() - app.Routes() +// TestServerHealth checks if the Echo server initializes properly +func TestServerHealth(t *testing.T) { + // Configure the location of the test profiles file + if err := os.Setenv(config.ProfilesFile, "testdata/test_profiles.json"); err != nil { + t.Fatalf("error setting env PROFILES_FILE: %v", err) + } + defer func() { + err := os.Unsetenv(config.ProfilesFile) + require.NoError(t, err) + }() - // Create a response recorder to record the response + engine, err := validation.NewEngine() + assert.NoError(t, err) + + service := &Service{Engine: engine} + server := echo.New() + server.Use(config.ZapLoggerMiddleware(engine.GetLogger())) + + // Register handlers + api.RegisterHandlers(server, service) + + // Perform a simple request to check if the server is functional + req := httptest.NewRequest(http.MethodGet, "/status", nil) rec := httptest.NewRecorder() - // Create a GET test request - req, err := http.NewRequest("GET", "/", nil) - assert.NoError(t, err, "Failed to create request") + // Serve the request + server.ServeHTTP(rec, req) + + // Server should respond with a 200 status + assert.Equal(t, http.StatusOK, rec.Code) +} + +// TestStatusEndpoint checks if the /status endpoint returns the expected JSON response +func TestStatusEndpoint(t *testing.T) { + // Configure the location of the test profiles file + if err := os.Setenv(config.ProfilesFile, "testdata/test_profiles.json"); err != nil { + t.Fatalf("error setting env PROFILES_FILE: %v", err) + } + defer func() { + err := os.Unsetenv(config.ProfilesFile) + require.NoError(t, err) + }() + + engine, err := validation.NewEngine() + assert.NoError(t, err) + + service := &Service{Engine: engine} + server := echo.New() + server.Use(config.ZapLoggerMiddleware(engine.GetLogger())) - // Call the handler directly - app.Router.ServeHTTP(rec, req) + // Register handlers + api.RegisterHandlers(server, service) - // Check the response status code and body - assert.Equal(t, http.StatusOK, rec.Code, "Expected status code 200") - assert.Equal(t, "hello world", rec.Body.String(), "Unexpected response body") + // Create a test request + request := httptest.NewRequest(http.MethodGet, "/status", nil) + recorder := httptest.NewRecorder() - // Create a POST test request - req, err = http.NewRequest("POST", "/", nil) + // Execute request + server.ServeHTTP(recorder, request) - // A new recorder must be created - rec = httptest.NewRecorder() - assert.NoError(t, err, "Failed to create request") - app.Router.ServeHTTP(rec, req) - assert.Equal(t, http.StatusNotFound, rec.Code, "Expected status code 404") + // Assertions + assert.Equal(t, http.StatusOK, recorder.Code) + assert.JSONEq(t, `{"service":"ok", "s3":"ok", "fs":"ok"}`, recorder.Body.String()) } diff --git a/openapi.yml b/openapi.yml new file mode 100644 index 0000000..119aa25 --- /dev/null +++ b/openapi.yml @@ -0,0 +1,129 @@ +openapi: 3.0.4 +info: + title: Validation Service API + description: A validation service that checks CSV files used in our digital workflow. + version: 0.0.1 +#servers: +# - url: 'https://validator.library.ucla.edu/' +paths: + /status: + get: + summary: Gets the validation service's current status + description: This endpoint returns a JSON object with information about the status of the service. + operationId: getStatus + responses: + '200': + $ref: '#/components/responses/StatusOK' + '500': + $ref: '#/components/responses/InternalServerError' + /status/{jobID}: + get: + description: View the requested job's status + operationId: getJobStatus + parameters: + - $ref: '#/components/parameters/JobIDParam' + responses: + '200': + $ref: '#/components/responses/StatusOK' + '404': + $ref: '#/components/responses/NotFoundError' + '500': + $ref: '#/components/responses/InternalServerError' + /upload/csv: + post: + summary: Start a new validation process + description: | + This endpoint starts a new validation process using the supplied profile and CSV upload + operationId: uploadCSV + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + required: + - csvFile + - profile + properties: + csvFile: + type: string + format: binary + description: The CSV file to be uploaded + profile: + type: string + description: The name of the profile the validation process should use + responses: + '201': + $ref: '#/components/responses/StatusCreated' + '404': + $ref: '#/components/responses/NotFoundError' + '500': + $ref: '#/components/responses/InternalServerError' + +components: + parameters: + ProfileIDParam: + name: profileID + in: path + required: true + schema: + type: string + description: The ID of the profile to use + JobIDParam: + name: jobID + in: path + required: true + schema: + type: string + description: An ID for the validation job + + schemas: + Status: + description: A JSON document representing the service's runtime status. It's intentionally brief, for now. + type: object + properties: + service: + type: string + example: ok + s3: + type: string + example: ok + fs: + type: string + example: ok + x-go-name: FS + required: + - service + - s3 + - fs + + responses: + StatusOK: + description: A response that returns a JSON object with status information + content: + application/json: + schema: + $ref: '#/components/schemas/Status' + StatusCreated: + description: A response indicating the requested resource has been created + content: + application/json: + schema: + $ref: '#/components/schemas/Status' + StatusNoContent: + description: A response that successfully acknowledges a request has been completed + content: {} + NotFoundError: + description: The requested resource was not able to be found + content: + text/plain: + schema: + type: string + example: The requested resource 'MyResource' could not be found + InternalServerError: + description: There was an internal server error + content: + text/plain: + schema: + type: string + example: The status request failed because the slithy toves did gyre diff --git a/testdata/cct-collection.csv b/testdata/cct-collection.csv index 4baa933..4f079b7 100644 --- a/testdata/cct-collection.csv +++ b/testdata/cct-collection.csv @@ -1,2 +1,2 @@ "Object Type","Item ARK","Parent ARK","File Name","Title","AltTitle.other","Visibility","Item Sequence","Archival Collection Title","Date.created","Date.normalized","Contents note","Description.note","Summary","Format.dimensions","Format.extent","Format.medium","Genre","Language","Local identifier","Name.creator","Name.subject","Name.artist","Repository","Type.typeOfResource","Rights.copyrightStatus","Rights.publicationStatus","Rights.rightsHolderContact","Description.fundingNote","Statement of Responsibility","Program","Subject geographic","Subject temporal","Subject","IIIF Manifest URL" -"Collection","ark:/21198/z1cz7hzc","","","Centro Cultural Tallersol (Chile)","","open","","Colección Afiches Antonio Kadima","1977-1994","1977/1994","Additional posters from Centro Cultural Tallersol will be digitized and published in the years ahead.","The archive at the Tallersol Cultural Center (Santiago, Chile) contains over eight thousand printed items that were produced by the graphic workshop between 1973 and the present day. Around five thousand of the pieces are posters. These were printed in colour and black-and-white by typography, screen printing, xylography (woodcut relief) and offset printing techniques on paper of various weights, formats and sizes, such as letter, legal, double letter, double legal and ‘mercurio’. The remaining three thousand items are cards, postcards, bulletins, event invitations, pamphlets, leaflets, books and poetry. Tallersol was the most productive graphic workshop in Santiago and the most enduring, as it continues its campaigns to commemorate resistance to dictatorship today. This digital collection includes an initial set of 150 posters, all created by artist Antonio Kadima.","El Centro Cultural Tallersol fue fundado en 1977 en Santiago de Chile por un colectivo de artistas y activistas culturales y políticos que resistieron la represión militar creando un espacio para la creatividad y experimentación artística bajo la dictadura de Pinochet de 1973-1990. En ese periodo, produjeron alrededor de nueve mil afiches, folletos y otros documentos para organizaciones de derechos humanos, políticas, sociales y culturales. Hasta ahora, el papel del arte gráfico en las redes de resistencia contracultural durante la dictadura es poco conocido y estudiado. Por eso, nuestro proyecto ha iniciado el trabajo de catalogar y digitalizar el archivo de Tallersol para democratizar el acceso al público e investigadores. // The Tallersol Cultural Center was founded in 1977 in Santiago de Chile by a collective of artists and cultural and political activists who resisted military repression by creating a space for creativity and artistic experimentation under the Pinochet dictatorship from 1973-1990. In that period, they produced around 9,000 posters, brochures and other documents for human rights, political, social and cultural organizations. Until now, the role of graphic art in countercultural resistance networks during the dictatorship is little known and studied.","","","","posters","spa","","Taller de Gráfica (Graphic Design Workshop)","","Antonio Kadima","El Centro Cultural Tallersol","","copyrighted","unpublished","antonio.kadima@gmail.com","Digitization for the Centro Cultural Tallersol collection was sponsored by the Modern Endangered Archives Program with funding from Arcadia.","","Modern Endangered Archives Program","Chile","","Political posters, Chilean--History--20th century|~|Government, Resistance to--Posters--Chile|~|Graphic arts--Chile--History--20th century|~|Affiches politiques chiliennes--Histoire--20e siècle|~|Résistance au gouvernement--Affiches--Chili","https://iiif.library.ucla.edu/collections/ark%3A%2F21198%2Fz1cz7hzc" +"Collection","ark:/21198/z1cz7hzc","","","Centro Cultural Tallersol (Chile)","","open","","Colección Afiches Antonio Kadima","1977-1994","1977/1994","Additional posters from Centro Cultural Tallersol will be digitized and published in the years ahead.","The archive at the Tallersol Cultural Center (Santiago, Chile) contains over eight thousand printed items that were produced by the graphic workshop between 1973 and the present day. Around five thousand of the pieces are posters. These were printed in colour and black-and-white by typography, screen printing, xylography (woodcut relief) and offset printing techniques on paper of various weights, formats and sizes, such as letter, legal, double letter, double legal and ‘mercurio’. The remaining three thousand items are cards, postcards, bulletins, event invitations, pamphlets, leaflets, books and poetry. Tallersol was the most productive graphic workshop in Santiago and the most enduring, as it continues its campaigns to commemorate resistance to dictatorship today. This digital collection includes an initial set of 150 posters, all created by artist Antonio Kadima.","El Centro Cultural Tallersol fue fundado en 1977 en Santiago de Chile por un colectivo de artistas y activistas culturales y políticos que resistieron la represión militar creando un espacio para la creatividad y experimentación artística bajo la dictadura de Pinochet de 1973-1990. En ese periodo, produjeron alrededor de nueve mil afiches, folletos y otros documentos para organizaciones de derechos humanos, políticas, sociales y culturales. Hasta ahora, el papel del arte gráfico en las redes de resistencia contracultural durante la dictadura es poco conocido y estudiado. Por eso, nuestro proyecto ha iniciado el trabajo de catalogar y digitalizar el archivo de Tallersol para democratizar el acceso al público echo investigadores. // The Tallersol Cultural Center was founded in 1977 in Santiago de Chile by a collective of artists and cultural and political activists who resisted military repression by creating a space for creativity and artistic experimentation under the Pinochet dictatorship from 1973-1990. In that period, they produced around 9,000 posters, brochures and other documents for human rights, political, social and cultural organizations. Until now, the role of graphic art in countercultural resistance networks during the dictatorship is little known and studied.","","","","posters","spa","","Taller de Gráfica (Graphic Design Workshop)","","Antonio Kadima","El Centro Cultural Tallersol","","copyrighted","unpublished","antonio.kadima@gmail.com","Digitization for the Centro Cultural Tallersol collection was sponsored by the Modern Endangered Archives Program with funding from Arcadia.","","Modern Endangered Archives Program","Chile","","Political posters, Chilean--History--20th century|~|Government, Resistance to--Posters--Chile|~|Graphic arts--Chile--History--20th century|~|Affiches politiques chiliennes--Histoire--20e siècle|~|Résistance au gouvernement--Affiches--Chili","https://iiif.library.ucla.edu/collections/ark%3A%2F21198%2Fz1cz7hzc" diff --git a/validation/checks/ark_check_test.go b/validation/checks/ark_check_test.go index d1bd364..4583a0e 100644 --- a/validation/checks/ark_check_test.go +++ b/validation/checks/ark_check_test.go @@ -1,12 +1,21 @@ package checks import ( + "flag" + "fmt" + "github.com/UCLALibrary/validation-service/testflags" "testing" "github.com/stretchr/testify/assert" "go.uber.org/multierr" ) +// TestCheck loads the flags for the tests in the 'check' package. +func TestCheck(t *testing.T) { + flag.Parse() + fmt.Printf("%s's log level: %s\n", t.Name(), *testflags.LogLevel) +} + // TestVerifyArk checks if verifyArk throws the correct errors when given incorrect ARKs func TestVerifyArk(t *testing.T) { tests := []struct { @@ -79,8 +88,8 @@ func TestVerifyArk(t *testing.T) { assert.Error(t, err) // If expectedErr is a combined error, check each error individually - if merr, ok := tt.expectedErr.(interface{ Unwrap() []error }); ok { - for _, expectedErr := range merr.Unwrap() { + if multiErr, ok := tt.expectedErr.(interface{ Unwrap() []error }); ok { + for _, expectedErr := range multiErr.Unwrap() { assert.ErrorIs(t, err, expectedErr, "expected error: %v", expectedErr) } } else { diff --git a/validation/checks/eol_check_test.go b/validation/checks/eol_check_test.go index b683783..cd6ad4a 100644 --- a/validation/checks/eol_check_test.go +++ b/validation/checks/eol_check_test.go @@ -3,19 +3,10 @@ package checks import ( - "flag" - "fmt" csv "github.com/UCLALibrary/validation-service/csvutils" - "github.com/UCLALibrary/validation-service/testflags" "testing" ) -// TestCheck loads the flags for the tests in the 'check' package. -func TestCheck(t *testing.T) { - flag.Parse() - fmt.Printf("%s's log level: %s\n", t.Name(), *testflags.LogLevel) -} - // TestEOLCheck_Validate tests the Validate method on EOLCheck. func TestEOLCheck_Validate(t *testing.T) { check := &EOLCheck{} diff --git a/validation/config/logger.go b/validation/config/logger.go new file mode 100644 index 0000000..21f3498 --- /dev/null +++ b/validation/config/logger.go @@ -0,0 +1,33 @@ +// Package config provides resources useful in the configuration of the validation service. +// +// This file contains a wrapper to configure Echo to use a Zap logger instead of the default. +package config + +import ( + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "go.uber.org/zap" +) + +// ZapLoggerMiddleware integrates the Zap logging into Echo and logs all requests +func ZapLoggerMiddleware(aLogger *zap.Logger) echo.MiddlewareFunc { + return middleware.RequestLoggerWithConfig(middleware.RequestLoggerConfig{ + LogURI: true, + LogStatus: true, + LogMethod: true, + LogLatency: true, + LogRemoteIP: true, + LogUserAgent: true, + LogValuesFunc: func(c echo.Context, values middleware.RequestLoggerValues) error { + aLogger.Debug("Request", + zap.String("method", values.Method), + zap.String("uri", values.URI), + zap.Int("status", values.Status), + zap.String("remote_ip", values.RemoteIP), + zap.String("user_agent", values.UserAgent), + zap.Duration("latency", values.Latency), + ) + return nil + }, + }) +} diff --git a/validation/config/logger_test.go b/validation/config/logger_test.go new file mode 100644 index 0000000..5b35286 --- /dev/null +++ b/validation/config/logger_test.go @@ -0,0 +1,63 @@ +//go:build unit + +// Package config provides resources useful in the configuration of the validation service. +// +// This file contains a test to confirm the Zap middleware is working with the Echo server. +package config + +import ( + "flag" + "fmt" + "github.com/UCLALibrary/validation-service/testflags" + "go.uber.org/zap" + "net/http" + "net/http/httptest" + "testing" + + "github.com/labstack/echo/v4" + "github.com/stretchr/testify/assert" +) + +// TestConfig loads the flags for the tests in the 'config' package. +func TestConfig(t *testing.T) { + flag.Parse() + fmt.Printf("%s's log level: %s\n", t.Name(), *testflags.LogLevel) +} + +// TestZapLoggerMiddleware ensures the Zap middleware logs requests correctly +func TestZapLoggerMiddleware(t *testing.T) { + logger := newTestLogger(t) + server := echo.New() + server.Use(ZapLoggerMiddleware(logger)) + + // Define a test route + server.GET("/test", func(c echo.Context) error { + return c.String(http.StatusOK, "test response") + }) + + // Create a test request + request := httptest.NewRequest(http.MethodGet, "/test", nil) + request.Header.Set("User-Agent", "TestAgent") + recorder := httptest.NewRecorder() + + // Execute middleware and handler + server.ServeHTTP(recorder, request) + + // Test the logger via the recorder + assert.Equal(t, http.StatusOK, recorder.Code) +} + +// newTestLogger gets a new logger to use in package's tests. +func newTestLogger(t *testing.T) *zap.Logger { + config := zap.NewDevelopmentConfig() + config.Level = zap.NewAtomicLevelAt(testflags.GetLogLevel()) + config.OutputPaths = []string{"stdout"} + config.ErrorOutputPaths = []string{"stderr"} + + logger, err := config.Build() + if err != nil { + t.Fatalf("Failed to build test logger: %v", err) + } + + return logger +} diff --git a/validation/config/profiles.go b/validation/config/profiles.go index 4f22138..4a38a07 100644 --- a/validation/config/profiles.go +++ b/validation/config/profiles.go @@ -33,7 +33,7 @@ import ( ) // LogLevel is the ENV property for the validation engine's configurable logging level. -const LogLevel string = "ZAP_LOG_LEVEL" +const LogLevel string = "LOG_LEVEL" // ProfilesFile is the ENV property for the location of the persisted JSON Profiles file. const ProfilesFile string = "PROFILES_FILE" @@ -105,11 +105,8 @@ func (profiles *Profiles) Refresh() error { if err != nil { return fmt.Errorf("failed to open '%s' file: %w", filePath, err) } - defer func() { - if closeErr := file.Close(); closeErr != nil { - fmt.Printf("warning: failed to close file '%s': %v\n", filePath, closeErr) - } - }() + //noinspection GoUnhandledErrorResult + defer file.Close() // Decode the persisted JSON file into a ProfilesSnapshot decoder := json.NewDecoder(file) @@ -235,8 +232,8 @@ func (profile *Profile) AddValidation(validation string) { profile.mutex.Lock() defer profile.mutex.Unlock() - for _, v := range profile.validations { - if v == validation { + for _, profileValidation := range profile.validations { + if profileValidation == validation { return // Skip duplicates } } @@ -273,13 +270,14 @@ func (profile *Profile) SetValidations(validations []string) { profile.lastUpdate = time.Now() uniqueValidations := make(map[string]struct{}) - for _, v := range validations { - uniqueValidations[v] = struct{}{} + + for _, validation := range validations { + uniqueValidations[validation] = struct{}{} } profile.validations = make([]string, 0, len(uniqueValidations)) - for v := range uniqueValidations { - profile.validations = append(profile.validations, v) + for validation := range uniqueValidations { + profile.validations = append(profile.validations, validation) } // Sort for consistency diff --git a/validation/config/profiles_test.go b/validation/config/profiles_test.go index 3f0a467..678f8f3 100644 --- a/validation/config/profiles_test.go +++ b/validation/config/profiles_test.go @@ -5,9 +5,7 @@ package config import ( "bytes" "encoding/json" - "flag" "fmt" - "github.com/UCLALibrary/validation-service/testflags" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "log" @@ -16,12 +14,6 @@ import ( "testing" ) -// TestConfig loads the flags for the tests in the 'config' package. -func TestConfig(t *testing.T) { - flag.Parse() - fmt.Printf("%s's log level: %s\n", t.Name(), *testflags.LogLevel) -} - // TestProfiles tests basic Get/Set functionality. func TestProfiles(t *testing.T) { profiles := NewProfiles() diff --git a/validation/engine.go b/validation/engine.go index a8b4539..0e4680f 100644 --- a/validation/engine.go +++ b/validation/engine.go @@ -33,6 +33,16 @@ func NewEngine(suppliedLogger ...*zap.Logger) (*Engine, error) { return nil, err } } + defer func() { + if syncErr := logger.Sync(); syncErr != nil { + // Combine the deferred sync error with a pre-existing existing error + if err == nil { + err = fmt.Errorf("error syncing logger: %w", syncErr) + } else { + err = fmt.Errorf("error syncing logger: %v; %w", syncErr, err) + } + } + }() // Create a new Profiles instance and load its persisted data from disk profiles := config.NewProfiles() @@ -81,19 +91,22 @@ func (engine *Engine) GetValidators(profileNames ...string) ([]Validator, error) // For each profile we've passed in... (the most common case will just be one) for _, profileName := range profileNames { profile := engine.profiles.GetProfile(profileName) - validations := removeExisting(profile.GetValidations(), existing) - // Right now, we're just passing Profiles as arguments to validator constructors - validators, err := engine.registry.GetValidators(validations, engine.profiles) - if err != nil { - return nil, err - } + if profile != nil { + validations := removeExisting(profile.GetValidations(), existing) + + // Right now, we're just passing Profiles as arguments to validator constructors + validators, err := engine.registry.GetValidators(validations, engine.profiles) + if err != nil { + return nil, err + } - // In the case of profiles containing the same checks, we only want to add a check once - for index, validatorName := range validators.Names { - if _, exists := existing[validatorName]; !exists { - existing[validatorName] = struct{}{} - checks = append(checks, validators.Checks[index]) + // In the case of profiles containing the same checks, we only want to add a check once + for index, validatorName := range validators.Names { + if _, exists := existing[validatorName]; !exists { + existing[validatorName] = struct{}{} + checks = append(checks, validators.Checks[index]) + } } } } @@ -108,16 +121,15 @@ func (engine *Engine) Validate(profile string, csvData [][]string) error { return fmt.Errorf("failed to get validators: %w", err) } + // Check to see if we have validators associated with the supplied profile + if len(validators) == 0 { + return fmt.Errorf("no validators found for profile: %s", profile) + } + // Have each validator check each cell in the supplied csvData for _, validator := range validators { for rowIndex, row := range csvData { - for colIndex, cell := range row { - engine.logger.Debug("Checking CSV data cell", - zap.Int("row", rowIndex), - zap.Int("col", colIndex), - zap.String("value", cell), - ) - + for colIndex, _ := range row { // Validate the data cell we're on, passing the entire CSV data matrix for additional context err := validator.Validate(profile, csv.Location{RowIndex: rowIndex, ColIndex: colIndex}, csvData) if err != nil { @@ -149,8 +161,12 @@ func buildLogger() (*zap.Logger, error) { loggerConfig := zap.NewProductionConfig() loggerConfig.Sampling = nil // We want to see all the things we log at a given level + // Explicitly set logs to be written to stdout instead of stderr + loggerConfig.OutputPaths = []string{"stdout"} + loggerConfig.ErrorOutputPaths = []string{"stderr"} + // In the production code, we just care about the ENV settings, not arg flags - logLevel := os.Getenv("ZAP_LOG_LEVEL") + logLevel := os.Getenv("LOG_LEVEL") // Set the log level based on the flag or environment variable switch logLevel { @@ -171,16 +187,6 @@ func buildLogger() (*zap.Logger, error) { if err != nil { return nil, fmt.Errorf("failed to initialize logger: %w", err) } - defer func() { - if syncErr := logger.Sync(); syncErr != nil { - // Combine the deferred sync error with a pre-existing existing error - if err == nil { - err = fmt.Errorf("error syncing logger: %w", syncErr) - } else { - err = fmt.Errorf("error syncing logger: %v; %w", syncErr, err) - } - } - }() return logger, nil } From 7d75e2d95e88cff34182aa67fc49f20ffe760a00 Mon Sep 17 00:00:00 2001 From: "Kevin S. Clarke" Date: Mon, 24 Feb 2025 15:50:10 -0500 Subject: [PATCH 2/7] Install oapi-codegen on GitHub Actions --- .github/workflows/build.yml | 4 ++++ .github/workflows/nightly.yml | 4 ++++ .github/workflows/release.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ba37a59..0f003ff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,6 +37,10 @@ jobs: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ sh -s -- -b $(go env GOPATH)/bin ${{ env.GO_LINTER_VERSION }} + - name: Install oapi-codegen + run: | + go install github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@latest + - name: Set up Docker Buildx uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d05107e..24fe949 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -32,6 +32,10 @@ jobs: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ sh -s -- -b $(go env GOPATH)/bin ${{ env.GO_LINTER_VERSION }} + - name: Install oapi-codegen + run: | + go install github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@latest + - name: Set up Docker Buildx uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4daea82..acbf70f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,6 +31,10 @@ jobs: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ sh -s -- -b $(go env GOPATH)/bin ${{ env.GO_LINTER_VERSION }} + - name: Install oapi-codegen + run: | + go install github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@latest + - name: Set up Docker Buildx uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0 From 21c4c960a07d84313ab44972731fd039bc9ad094 Mon Sep 17 00:00:00 2001 From: "Kevin S. Clarke" Date: Fri, 28 Feb 2025 16:19:21 -0500 Subject: [PATCH 3/7] [SERV-1222] WIP --- .gitignore | 5 +- Dockerfile | 3 + Makefile | 38 +++- README.md | 13 +- api/api.go | 64 ++---- html/assets/.keep | 3 + html/template/index.tmpl | 136 ++++++++++++ main.go | 198 ++++++++++++++---- openapi.yml | 27 +-- profiles.example.json | 15 ++ validation/checks/ark_check.go | 2 +- validation/checks/eol_check.go | 2 +- validation/checks/eol_check_test.go | 2 +- validation/engine.go | 2 +- validation/engine_test.go | 4 +- {csvutils => validation/utils}/csv_io.go | 4 +- {csvutils => validation/utils}/csv_io_test.go | 2 +- {csvutils => validation/utils}/location.go | 2 +- .../utils}/location_test.go | 2 +- validation/validator.go | 4 +- validation/validator_test.go | 2 +- 21 files changed, 401 insertions(+), 129 deletions(-) create mode 100644 html/assets/.keep create mode 100644 html/template/index.tmpl create mode 100644 profiles.example.json rename {csvutils => validation/utils}/csv_io.go (97%) rename {csvutils => validation/utils}/csv_io_test.go (99%) rename {csvutils => validation/utils}/location.go (98%) rename {csvutils => validation/utils}/location_test.go (99%) diff --git a/.gitignore b/.gitignore index 831b937..ef9110c 100644 --- a/.gitignore +++ b/.gitignore @@ -26,5 +26,8 @@ go.work # Build artifacts *-service -# A profile configuration for the validation service +# The automatically duplicated version of the OpenAPI spec file +html/assets/openapi.yml + +# A real configuration file that our server will use when running locally profiles.json diff --git a/Dockerfile b/Dockerfile index 47590c5..b884b78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,6 +54,9 @@ RUN addgroup -S "${SERVICE_NAME}" && adduser -S "${SERVICE_NAME}" -G "${SERVICE_ # Create a directory for our profiles file RUN mkdir -p "$DATA_DIR" +# Copy the templates directory into our container +COPY "html/" "$DATA_DIR/html/" + # Copy the file without --chown or --chmod (BuildKit not required) COPY --from=build "/${SERVICE_NAME}" "/sbin/${SERVICE_NAME}" COPY "testdata/test_profiles.json" "$PROFILES_FILE" diff --git a/Makefile b/Makefile index 4a50196..3e0bfcd 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,36 @@ -# Build and runtime variables +# Build and runtime variables that can be overridden SERVICE_NAME := validation-service LOG_LEVEL := info PORT := 8888 +# Force the API target to run even if the openapi.yml has not been touched/changed +ifneq ($(filter FORCE,$(MAKECMDGOALS)),) +.PHONY: api/api.go +endif + +# Define FORCE as a target so accidentally using on other targets won't cause errors +.PHONY: FORCE +FORCE: + @echo "Makefile target(s) run with FORCE to require an API code rebuild" + # Do a full build of the project -all: api lint build test docker-test +all: config api lint build test docker-test -# Lint the code +# Lint the code for correctness lint: golangci-lint run -# Generate Go code from the OpenAPI specification only when it has changed +# We generate Go API code from the OpenAPI specification only when it has changed +# We assume Windows developers are using WSL, so we don't define $(CP) for this api/api.go: openapi.yml oapi-codegen -package api -generate types,server,spec -o api/api.go openapi.yml + cp openapi.yml html/assets/openapi.yml # This is an alias for the longer API generation Makefile target api/api.go api: api/api.go # Build the Go project -build: +build: api go build -o $(SERVICE_NAME) # Run Go tests, excluding tests in the 'integration' directory @@ -31,7 +43,8 @@ docker-build: # A convenience target to assist with running the Docker container outside of the build (optional) docker-run: - docker run -p $(PORT):8888 --name $(SERVICE_NAME) -d $(shell docker image ls -q --filter=reference=$(SERVICE_NAME)) + CONTAINER_ID=$(shell docker image ls -q --filter=reference=$(SERVICE_NAME)); \ + docker run -p $(PORT):8888 --name $(SERVICE_NAME) -e LOG_LEVEL="$(LOG_LEVEL)" -d $$CONTAINER_ID docker-logs: docker logs -f $(shell docker ps --filter "name=$(SERVICE_NAME)" --format "{{.ID}}") @@ -48,6 +61,13 @@ docker-test: clean: rm -rf $(SERVICE_NAME) api/api.go -# Run the validation service locally -run: api build - PROFILES_FILE="testdata/test_profiles.json" ./$(SERVICE_NAME) +# Creates a new local profile configuration file if it doesn't already exist +profile.json: profiles.example.json + @if [ ! -f profile.json ]; then cp profiles.example.json profiles.json; fi + +# An alias for the profile.json target +config: profile.json + +# Run the validation service locally, independent of the Docker container +run: config api build + PROFILES_FILE="profiles.json" LOG_LEVEL=$(LOG_LEVEL) ./$(SERVICE_NAME) diff --git a/README.md b/README.md index 60a5906..e72605f 100644 --- a/README.md +++ b/README.md @@ -60,10 +60,21 @@ Note: If you want to change the values defined in the Makefile (echo.g., the `LO make test LOG_LEVEL=debug -To run the validation service, locally, for testing purposes: +To run the validation service, without a Docker container, for live testing purposes (i.e., the fastest way to test): make run +or + + make run LOG_LEVEL=debug + +The `run` or `all` targets can also be run with `FORCE` to force the API code to be regenerated, even if the OpenAPI +spec hasn't changed since the last run: + + make run LOG_LEVEL=debug FORCE + +The usual behavior of `run` or `all` is not to run the `api` target if the OpenAPI spec has not been touched/changed. + ### Working with Docker One can also run Docker locally, for more hands-on testing, from the Makefile. Unlike the tests, which will not leave diff --git a/api/api.go b/api/api.go index d4d9b16..7f4b810 100644 --- a/api/api.go +++ b/api/api.go @@ -8,27 +8,22 @@ import ( "compress/gzip" "encoding/base64" "fmt" - "net/http" "net/url" "path" "strings" "github.com/getkin/kin-openapi/openapi3" "github.com/labstack/echo/v4" - "github.com/oapi-codegen/runtime" openapi_types "github.com/oapi-codegen/runtime/types" ) // Status A JSON document representing the service's runtime status. It's intentionally brief, for now. type Status struct { - FS string `json:"fs"` - S3 string `json:"s3"` - Service string `json:"service"` + Fester string `json:"fester"` + FileSystem string `json:"filesystem"` + Service string `json:"service"` } -// JobIDParam defines model for JobIDParam. -type JobIDParam = string - // StatusCreated A JSON document representing the service's runtime status. It's intentionally brief, for now. type StatusCreated = Status @@ -52,10 +47,7 @@ type ServerInterface interface { // Gets the validation service's current status // (GET /status) GetStatus(ctx echo.Context) error - - // (GET /status/{jobID}) - GetJobStatus(ctx echo.Context, jobID JobIDParam) error - // Start a new validation process + // Uploads and validates CSV files // (POST /upload/csv) UploadCSV(ctx echo.Context) error } @@ -74,22 +66,6 @@ func (w *ServerInterfaceWrapper) GetStatus(ctx echo.Context) error { return err } -// GetJobStatus converts echo context to params. -func (w *ServerInterfaceWrapper) GetJobStatus(ctx echo.Context) error { - var err error - // ------------- Path parameter "jobID" ------------- - var jobID JobIDParam - - err = runtime.BindStyledParameterWithOptions("simple", "jobID", ctx.Param("jobID"), &jobID, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter jobID: %s", err)) - } - - // Invoke the callback with all the unmarshaled arguments - err = w.Handler.GetJobStatus(ctx, jobID) - return err -} - // UploadCSV converts echo context to params. func (w *ServerInterfaceWrapper) UploadCSV(ctx echo.Context) error { var err error @@ -128,7 +104,6 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL } router.GET(baseURL+"/status", wrapper.GetStatus) - router.GET(baseURL+"/status/:jobID", wrapper.GetJobStatus) router.POST(baseURL+"/upload/csv", wrapper.UploadCSV) } @@ -136,22 +111,21 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/7xWTW/jRgz9K8S0QC6K5e2mF9/SbFM4RXcXdZrLdg8jDW1NIs2oQ8qOEfi/Fxx9+CNK", - "E3SBvdnSzOPj4yOpJ5X7qvYOHZOaPalaB10hY4j/bnw2//BZHsk/g5QHW7P1Ts3UpYP5B1j6AFwgrHVp", - "jZZXcO8zlSgrZ2rNhUqU0xWqmboXNJWogP80NqBRMw4NJoryAistEXhby0HiYN1K7XY7OUy1d4SRz9wx", - "BqfLBYY1hl9D8EEe594xOo4I+MhpXWoJ/3SAjI+6qksBvy0QiDU3BEIEiWGpbYkGMsx1QxjzodJysQX2", - "ayQw1sBqG1AlYwyPZbktMCBsNIF2YDu+QJEwYGS8S9RHz9e+ceb/p9BxRwMByTchRzj7Y/tn9/sMct+U", - "BpxnyBCWEuuN7MeQJR2B0lmJwH4PuUvUImp5FVCzlPQoFV3Xpc2jLdJ78icJ/RhwqWbqh3RvwbR9S2mL", - "OkbxEnpLgHUmortVrNkI8UITZIgO8o7fQPjT79+XKxeaISA3wRFouFl8+gg+u8ecYWO56C1p3dKHKrJQ", - "gtfFEApdmOeN2IIZnzcVOglSByR0gy5iPpvjGUFoHNuq9/8E5nxG0aROsHRZbiELFpdJ7GvnNxOVqDr4", - "GgPbtgWXdOxF//DMV4l6PF/5867rrxciOr1/9Zqcapm+fnR3OEW+DPdinEQ4fh2utCK31RF1xwQ8mF4d", - "VFuvvMD8geBqcQdLWyJBQ2jAOvBNAGNXlnUJGx8elmWrFVuOnO/2gIsO8PLzXCVqjYHaqNPJdPJOkvY1", - "Ol1bNVPvJ9PJhSiuuYgypzTUfIX8nPltYQnQmdpb95/uOrAV6Mw33Bqj9ZxfHtpE0pCCx8Nzo2bqN+TO", - "eyfj+Kfp9KXeGM6lQ7vtEvXzWy6MDfnYCk1V6bBtCdHp1tmbPG9CkD6gvi2TXsf0Ka6g3Yt63lncnIyS", - "e5+dUY81osyNzwZxDpfnl/E090fSg+W6+/qtyl5ML16/cLx1vq0eUdamLr02aU7r+O3g6VWPEuvAYlGH", - "m8Py1cHnSNJgw9RqZCCjkVfSfKCdiZ3YBv3bPavGX/HF1eKu+8ZA4l+82Z7M+aop2dY6cCotcW406+NR", - "fzzvclpf2xLH8sJhMHQ7sWWGsmnbdlMzlVknph0Zdl1e48gyPPvG7AU4cXwvGRVx1TeEr87JPpl98PE5", - "efyFtntmzndvNefVfu1+d4ceToyF2O5F10U7/xsAAP//oXcG8woLAAA=", + "H4sIAAAAAAAC/7yV32/jRBDH/5XRgtQXX5yD8pK3o+JQQdwhcvQFeNh4x/Fe7V0zM5s0qvK/o1nbSdME", + "WiHEm3/sznznO5/ZfTRV7PoYMAibxaMh5D4GxvxyGwQp2HaJtEH6jiiSfq5iEAyij4IPUvat9UHfuGqw", + "s/qED7brWzQL86lBYLGSGAj/TMgCtfUtOlhhZRMjiK5ovTQ7kLhBBucdrHeEpjCy6zUIC/mwNvv9vjAO", + "uSLfi49hCE8IW8tgA/hRL3AWDJgV7wvzIcr7mIL79yWM2tEBIcdEFcLVT7tfxucrqGJqHYQosEKoNdcr", + "1V+KrOVoKLtqESQeQ+4Ls8xe3hBaQfesFNv3ra+sBi8/c3xW0JeEtVmYL8pjx8vhL5dD1EsS38GEBPjg", + "cvSwzj27ILyxDCvEANWo7yD444//r1ZprAChJAoMFn5YfvwAcfUZK4Gtl2ZC0oc6UpdVGI035lAJY5rF", + "41mSHMzFKnUYNElPyBgOvih8vsIrBkpBfDfxP4NbueIMadBYtm13sCKPdQF1JAhxOzOF6Sn2SOKHEazV", + "YTrlMd6fs1WY2rfIOxbsXlxdmIc36/gm2E4/vvctLoeN6sCg/uWM+8IoAZ4Uw98O+4pJ8omiPw7bhyYM", + "3VP3Lxm8sa13uSmTmUM/qware4ab5R3k2JAYHfgAMRE4v/ZiW9hGuq/buGU1U7zkAu6OEZdjxHc/35rC", + "bJB4SDufzWdv1YHYY7C9Nwvz9Ww+u9aWWGlyN0o+QLFGOZf+qfEMGFwfffhH/J5wB3YVkwzkDFDG+ilH", + "WoYSkRffOrMw36OMcBan5/VX8/nfDc9hXXmYx31hvnnNhku3QJ6V1HWWdoMgzpLPG3fFUCUiHRSe5rYw", + "ZerbaF1Z8Ubz95FfNJPFkqiXAbdP8/QUK2RF4TB/SY8WdPpLMQEbXGZmSPp7ODP01/zjZnlnBqaR5dvo", + "ds9OrC614ntLUmrv3jgr9vTQOp3cijc6WZfqwgPC4+k+KEO9MwYuzMKsfFB3L0z6WNflyDrUE0GTAc9a", + "M1nGTb60EuOL0z0Vc0x+eaKPe4QS7s/4fPtaPm+OF8j1/PrlXac3/H+H9oAGZ4ZGC/HJCaRV7/8KAAD/", + "/y+FZAlECQAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/html/assets/.keep b/html/assets/.keep new file mode 100644 index 0000000..04730be --- /dev/null +++ b/html/assets/.keep @@ -0,0 +1,3 @@ +* +* Keep this file so the assets directory will be persisted in git. +* diff --git a/html/template/index.tmpl b/html/template/index.tmpl new file mode 100644 index 0000000..d1ee89f --- /dev/null +++ b/html/template/index.tmpl @@ -0,0 +1,136 @@ +{{ define "index.html" }} + + + + + + CSV Validation + + + + + + + + + + + +
+
+
+

CSV Validator

+

+ Upload your CSV files for validation of their data and metadata  🚀 +

+ +
+ +
+
+
+ +
+ +
+
+ +
+ +
+ +
+

+ Specify the name of the validation profile to use with the uploaded CSV. +

+
+ +
+
+ +
+
+ +
version: {{ .Version }}
+
+
+ + +
+
+

Instructions

+
CSV Validator will check the data files and metadata in uploaded CSV files to confirm + there are not any issues that will cause problems when loading the CSV into any of our + workflow services. At the start, there will be a small number of validators. But, as we + encounter new issues with our workflow processes, additional validators will be added.
+

+ To start a batch job, upload a CSV file (like one of those found in the + Eureka GitHub repository). If you would like image + dimensions + looked up in a different IIIF server, enter the base URL in the text box. +

+
+
+
+
+ +
+
+ + + +{{ end }} diff --git a/main.go b/main.go index 7965bb2..c82e557 100644 --- a/main.go +++ b/main.go @@ -4,14 +4,18 @@ import ( "errors" "fmt" "github.com/UCLALibrary/validation-service/api" - "github.com/UCLALibrary/validation-service/csvutils" "github.com/UCLALibrary/validation-service/validation" "github.com/UCLALibrary/validation-service/validation/config" + "github.com/UCLALibrary/validation-service/validation/utils" "github.com/labstack/echo/v4" middleware "github.com/oapi-codegen/echo-middleware" "go.uber.org/zap" + "html/template" + "io" "log" "net/http" + "path/filepath" + "sync" ) // Port is the default port for our server @@ -22,23 +26,26 @@ type Service struct { Engine *validation.Engine } +// TemplateRegistry holds parsed HTML templates for the validation service's Web pages +type TemplateRegistry struct { + templates *template.Template + mu sync.Mutex +} + +// Render implements Echo's `Renderer` interface +func (tmplReg *TemplateRegistry) Render(writer io.Writer, name string, data interface{}, context echo.Context) error { + tmplReg.mu.Lock() + defer tmplReg.mu.Unlock() + return tmplReg.templates.ExecuteTemplate(writer, name, data) +} + // GetStatus handles the GET /status request func (service *Service) GetStatus(context echo.Context) error { // A placeholder response return context.JSON(http.StatusOK, api.Status{ - Service: "ok", - S3: "ok", - FS: "ok", - }) -} - -// GetJobStatus handles the GET /status/{jobID} request -func (service *Service) GetJobStatus(context echo.Context, jobID string) error { - // A placeholder response; structure will change - return context.JSON(http.StatusOK, api.Status{ - Service: "completed", - S3: "ok", - FS: "ok", + Service: "ok", + Fester: "ok", + FileSystem: "ok", }) } @@ -59,7 +66,7 @@ func (service *Service) UploadCSV(context echo.Context) error { zap.String("profile", profile)) // Parse the CSV data - csvData, readErr := csvutils.ReadUpload(file, logger) + csvData, readErr := utils.ReadUpload(file, logger) if readErr != nil { return context.JSON(http.StatusBadRequest, map[string]string{"error": "Uploaded CSV file could not be parsed"}) } @@ -67,66 +74,93 @@ func (service *Service) UploadCSV(context echo.Context) error { if err := engine.Validate(profile, csvData); err != nil { // Handle if there was a validation error return context.JSON(http.StatusCreated, api.Status{ - Service: fmt.Sprintf("error: %v", err), - S3: "ok", - FS: "ok", + Service: fmt.Sprintf("error: %v", err), + Fester: "ok", + FileSystem: "ok", }) } // Handle if there were no validation errors return context.JSON(http.StatusCreated, api.Status{ - Service: "created", - S3: "ok", - FS: "ok", + Service: "created", + Fester: "ok", + FileSystem: "ok", }) } // Main function starts our Echo server func main() { - // Create a new validation engine + // Create a new validation engine for our service to use engine, err := validation.NewEngine() if err != nil { log.Fatal(err) } - // Create the validation service - service := &Service{ - Engine: engine, - } + // Logger we can use to output information + logger := engine.GetLogger() // Create a new validation application and configure its logger echoApp := echo.New() echoApp.Use(config.ZapLoggerMiddleware(engine.GetLogger())) - // Hide the Echo startup messages + // Hide Echo startup messages that don't play nicely with logger echoApp.HideBanner = true echoApp.HidePort = true - // Handle requests with and without a trailing slash - echoApp.Pre(func(next echo.HandlerFunc) echo.HandlerFunc { - return func(char echo.Context) error { - path := char.Request().URL.Path + // Turn on Echo's debugging features if we're set to debug (mostly more info in errors) + if debugging := logger.Check(zap.DebugLevel, "Enable debugging"); debugging != nil { + echoApp.Debug = true + } - if path != "/" && path[len(path)-1] == '/' { - // Remove trailing slash as our canonical form - char.Request().URL.Path = path[:len(path)-1] - } + // Serves the service's OpenAPI specification at the expected endpoint + echoApp.GET("/openapi.yml", func(context echo.Context) error { + return context.File("html/assets/openapi.yml") + }) - return next(char) + // Sets the template renderer for the application + echoApp.Renderer = setTemplateRenderer(logger) + + // Handle our SPA endpoint outside of the OpenAPI specification + echoApp.GET("/", func(context echo.Context) error { + data := map[string]interface{}{ + "Version": "0.0.1", } + + return context.Render(http.StatusOK, "index.html", data) }) + // Handle requests with and without a trailing slash + echoApp.Pre(TrailingSlashMiddleware) + // Load the OpenAPI spec for request validation swagger, swaggerErr := api.GetSwagger() if swaggerErr != nil { log.Fatalf("Failed to load OpenAPI spec: %v", swaggerErr) } - // Register the Echo/OpenAPI validation middleware - echoApp.Use(middleware.OapiRequestValidator(swagger)) + // Register the Echo/OpenAPI validation middleware; have it ignore things served independent of the OpenAPI spec + echoApp.Use(middleware.OapiRequestValidatorWithOptions(swagger, &middleware.Options{ + Skipper: func(context echo.Context) bool { + return context.Path() == "/openapi.yml" || context.Path() == "/" + }, + })) + + // Register request handlers for our service + api.RegisterHandlers(echoApp, &Service{ + Engine: engine, + }) + + // Log the configured routes when we're running in debug mode + if debugging := logger.Check(zap.DebugLevel, "Loading routes"); debugging != nil { + var fields []zap.Field + + for _, route := range echoApp.Routes() { + routeInfo := []string{route.Method, route.Path} + fields = append(fields, zap.Strings("route", routeInfo)) + } - // Register request handlers - api.RegisterHandlers(echoApp, service) + logger.Debug("Registered routes", fields...) + } // Configure the validation echoApp server := &http.Server{ @@ -139,3 +173,87 @@ func main() { log.Fatalf("Server failed: %v", err) } } + +// TrailingSlashMiddleware handles paths with slashes at the end so they also resolve. +func TrailingSlashMiddleware(next echo.HandlerFunc) echo.HandlerFunc { + return func(context echo.Context) error { + path := context.Request().URL.Path + + // Strip trailing slashes if found in path + if path != "/" && path[len(path)-1] == '/' { + context.Request().URL.Path = path[:len(path)-1] + } + + return next(context) + } +} + +// SetTemplateRenderer loads the HTML templates and then provides a template registry that can render them. +func setTemplateRenderer(logger *zap.Logger) *TemplateRegistry { + templates, templateErr := loadTemplates(logger) + if templateErr != nil { + logger.Error(templateErr.Error()) + } + + // Log all the HTML templates that were loaded if we're in debugging mode + if debugging := logger.Check(zap.DebugLevel, "Load templates"); debugging != nil && templates != nil { + var fields []zap.Field + + for _, tmpl := range templates.Templates() { + // We add a 'root' logger with an empty name, but it does nothing so delete it + if tmpl.Name() != "" { + fields = append(fields, zap.String("template", tmpl.Name())) + } + } + + logger.Debug("Loaded Web resources", fields...) + } + + return &TemplateRegistry{templates: templates} +} + +// loadTemplates loads the available HTML templates for the Web UI. +func loadTemplates(logger *zap.Logger) (*template.Template, error) { + templates := template.New("") // New set of templates + + // We try both locations: the Docker container's and the local dev's + patterns := []string{ + // The templates should exist in only one of these locations + "/usr/local/data/html/template/*.tmpl", // Docker templates + "html/template/*.tmpl", // Local templates + } + + foundTemplates := false + + // Parse each template path pattern so we can add any matches to the template set + for _, pattern := range patterns { + // Check if any files match the pattern before parsing them + matches, err := filepath.Glob(pattern) + if err != nil { + return nil, fmt.Errorf("error checking pattern %s: %w", pattern, err) + } + + // Skip this pattern if no template files are found here + if len(matches) == 0 { + logger.Debug("No templates found (Skipping)", zap.String("location", pattern)) + continue // Moving on to the next location + } + + // If we get this far, at least one location had templates + foundTemplates = true + + // Attempt to parse the templates once we know they exist + templates, err = templates.ParseGlob(pattern) + if err != nil { + return nil, fmt.Errorf("error loading templates from %s: %w", pattern, err) + } + } + + // If no templates were found in either location, return an error + if !foundTemplates { + return nil, fmt.Errorf("no templates found in any of the specified locations") + } + + // Return the set of template matches + return templates, nil +} diff --git a/openapi.yml b/openapi.yml index 119aa25..cd55b2b 100644 --- a/openapi.yml +++ b/openapi.yml @@ -1,7 +1,7 @@ openapi: 3.0.4 info: title: Validation Service API - description: A validation service that checks CSV files used in our digital workflow. + description: A validation service that checks CSV files used in our digital workflows. version: 0.0.1 #servers: # - url: 'https://validator.library.ucla.edu/' @@ -16,22 +16,9 @@ paths: $ref: '#/components/responses/StatusOK' '500': $ref: '#/components/responses/InternalServerError' - /status/{jobID}: - get: - description: View the requested job's status - operationId: getJobStatus - parameters: - - $ref: '#/components/parameters/JobIDParam' - responses: - '200': - $ref: '#/components/responses/StatusOK' - '404': - $ref: '#/components/responses/NotFoundError' - '500': - $ref: '#/components/responses/InternalServerError' /upload/csv: post: - summary: Start a new validation process + summary: Uploads and validates CSV files description: | This endpoint starts a new validation process using the supplied profile and CSV upload operationId: uploadCSV @@ -85,17 +72,17 @@ components: service: type: string example: ok - s3: + fester: type: string example: ok - fs: + filesystem: type: string example: ok - x-go-name: FS + x-go-name: FileSystem required: - service - - s3 - - fs + - fester + - filesystem responses: StatusOK: diff --git a/profiles.example.json b/profiles.example.json new file mode 100644 index 0000000..304ad3a --- /dev/null +++ b/profiles.example.json @@ -0,0 +1,15 @@ +{ + "profiles": { + "example": { + "name": "example", + "lastUpdate": "2025-01-01T12:00:00Z", + "validations": ["Validation1", "Validation2"] + }, + "test": { + "name": "test", + "lastUpdate": "2025-01-10T15:30:00Z", + "validations": ["EOLCheck"] + } + }, + "lastUpdate": "2025-01-10T16:00:00Z" +} diff --git a/validation/checks/ark_check.go b/validation/checks/ark_check.go index 56cdb77..4412e68 100644 --- a/validation/checks/ark_check.go +++ b/validation/checks/ark_check.go @@ -3,10 +3,10 @@ package checks import ( "errors" "fmt" + csv "github.com/UCLALibrary/validation-service/validation/utils" "regexp" "strings" - csv "github.com/UCLALibrary/validation-service/csvutils" "github.com/UCLALibrary/validation-service/validation/config" "go.uber.org/multierr" ) diff --git a/validation/checks/eol_check.go b/validation/checks/eol_check.go index 6a77b0f..a1f9cda 100644 --- a/validation/checks/eol_check.go +++ b/validation/checks/eol_check.go @@ -2,8 +2,8 @@ package checks import ( "fmt" - csv "github.com/UCLALibrary/validation-service/csvutils" "github.com/UCLALibrary/validation-service/validation/config" + csv "github.com/UCLALibrary/validation-service/validation/utils" "strings" ) diff --git a/validation/checks/eol_check_test.go b/validation/checks/eol_check_test.go index cd6ad4a..2a6108e 100644 --- a/validation/checks/eol_check_test.go +++ b/validation/checks/eol_check_test.go @@ -3,7 +3,7 @@ package checks import ( - csv "github.com/UCLALibrary/validation-service/csvutils" + csv "github.com/UCLALibrary/validation-service/validation/utils" "testing" ) diff --git a/validation/engine.go b/validation/engine.go index 0e4680f..30345fd 100644 --- a/validation/engine.go +++ b/validation/engine.go @@ -5,8 +5,8 @@ package validation import ( "fmt" - csv "github.com/UCLALibrary/validation-service/csvutils" "github.com/UCLALibrary/validation-service/validation/config" + csv "github.com/UCLALibrary/validation-service/validation/utils" "go.uber.org/zap" "go.uber.org/zap/zapcore" "os" diff --git a/validation/engine_test.go b/validation/engine_test.go index cbbb9aa..fcb5f64 100644 --- a/validation/engine_test.go +++ b/validation/engine_test.go @@ -8,9 +8,9 @@ package validation import ( "flag" "fmt" - "github.com/UCLALibrary/validation-service/csvutils" "github.com/UCLALibrary/validation-service/testflags" "github.com/UCLALibrary/validation-service/validation/config" + "github.com/UCLALibrary/validation-service/validation/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" @@ -85,7 +85,7 @@ func TestEngine_Validate(t *testing.T) { } // Read in our CSV test data - csvData, csvErr := csvutils.ReadFile("../testdata/cct-works-simple.csv", engine.GetLogger()) + csvData, csvErr := utils.ReadFile("../testdata/cct-works-simple.csv", engine.GetLogger()) if csvErr != nil { require.NoError(t, csvErr) } diff --git a/csvutils/csv_io.go b/validation/utils/csv_io.go similarity index 97% rename from csvutils/csv_io.go rename to validation/utils/csv_io.go index 992cd62..571fd56 100644 --- a/csvutils/csv_io.go +++ b/validation/utils/csv_io.go @@ -1,7 +1,7 @@ -// Package csvutils provides tools to work with CSV data. +// Package utils provides utilities for the application. // // This file provides a reader and writer for CSV data. -package csvutils +package utils import ( "encoding/csv" diff --git a/csvutils/csv_io_test.go b/validation/utils/csv_io_test.go similarity index 99% rename from csvutils/csv_io_test.go rename to validation/utils/csv_io_test.go index 5713a96..13354a4 100644 --- a/csvutils/csv_io_test.go +++ b/validation/utils/csv_io_test.go @@ -3,7 +3,7 @@ // Package csvutils provides tools to work with CSV data. // // This file provides tests for readers and writers of CSV data. -package csvutils +package utils import ( "encoding/csv" diff --git a/csvutils/location.go b/validation/utils/location.go similarity index 98% rename from csvutils/location.go rename to validation/utils/location.go index a55a0bd..56d5b3b 100644 --- a/csvutils/location.go +++ b/validation/utils/location.go @@ -1,5 +1,5 @@ // Package csvutils has structures and utilities useful for working with CSVs. -package csvutils +package utils import ( "errors" diff --git a/csvutils/location_test.go b/validation/utils/location_test.go similarity index 99% rename from csvutils/location_test.go rename to validation/utils/location_test.go index 80024b3..ce3c265 100644 --- a/csvutils/location_test.go +++ b/validation/utils/location_test.go @@ -1,7 +1,7 @@ //go:build unit // Package csvutils has structures and utilities useful for working with CSVs. -package csvutils +package utils import ( "testing" diff --git a/validation/validator.go b/validation/validator.go index afc12a5..93feb69 100644 --- a/validation/validator.go +++ b/validation/validator.go @@ -3,7 +3,9 @@ // This file provides a Validator interface that individual checks should implement. package validation -import csv "github.com/UCLALibrary/validation-service/csvutils" +import ( + csv "github.com/UCLALibrary/validation-service/validation/utils" +) // Validator interface defines how implementations should be called. type Validator interface { diff --git a/validation/validator_test.go b/validation/validator_test.go index b7f26fc..32e2cc8 100644 --- a/validation/validator_test.go +++ b/validation/validator_test.go @@ -4,7 +4,7 @@ package validation import ( "fmt" - csv "github.com/UCLALibrary/validation-service/csvutils" + csv "github.com/UCLALibrary/validation-service/validation/utils" "testing" "github.com/stretchr/testify/assert" From 0925c9dbc4383b9fdd5503ce7ed2b2aa4214cbe3 Mon Sep 17 00:00:00 2001 From: "Kevin S. Clarke" Date: Fri, 28 Feb 2025 23:52:10 -0500 Subject: [PATCH 4/7] Add sidebar styles --- html/template/index.tmpl | 55 ++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/html/template/index.tmpl b/html/template/index.tmpl index d1ee89f..8a808a5 100644 --- a/html/template/index.tmpl +++ b/html/template/index.tmpl @@ -7,7 +7,7 @@ CSV Validation - + @@ -56,7 +90,7 @@