From ef137e2dd23e097e4cbaabb361c466891e854942 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Mon, 8 Apr 2024 15:22:32 +0300 Subject: [PATCH 01/13] Rename module and update docker file --- Dockerfile | 10 +++++++--- README.md | 6 +++--- activities/geoparquet_to_geojson.go | 4 ++-- go.mod | 2 +- internal/geojson/featurereader.go | 2 +- internal/geojson/featurereader_test.go | 4 ++-- internal/geojson/geojson.go | 6 +++--- internal/geojson/geojson_test.go | 8 ++++---- internal/geojson/recordwriter.go | 4 ++-- internal/geoparquet/featurewriter.go | 2 +- internal/geoparquet/geoparquet.go | 4 ++-- internal/geoparquet/geoparquet_test.go | 6 +++--- internal/geoparquet/metadata.go | 2 +- internal/pqutil/arrow.go | 2 +- internal/pqutil/arrow_test.go | 4 ++-- internal/pqutil/parquet_test.go | 4 ++-- internal/pqutil/transform_test.go | 6 +++--- internal/storage/blob_test.go | 2 +- internal/storage/http_test.go | 2 +- internal/storage/storage_test.go | 2 +- internal/test/test.go | 4 ++-- internal/validator/rules.go | 4 ++-- internal/validator/validator.go | 4 ++-- internal/validator/validator_test.go | 10 +++++----- main.go | 4 ++-- workflows/geoparquet.go | 2 +- 26 files changed, 57 insertions(+), 53 deletions(-) diff --git a/Dockerfile b/Dockerfile index c889004..f2aef3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,13 @@ -FROM golang:1.20 AS builder +FROM golang:1.20 AS base WORKDIR /app COPY . . -RUN go build -o ./bin/akuko-geoparquet-temporal-tooling +FROM base as production +RUN go build -o ./bin/akuko-temporal-go-tooling +ENTRYPOINT ["./bin/akuko-temporal-go-tooling"] -ENTRYPOINT ["./bin/akuko-geoparquet-temporal-tooling"] +FROM base as dev +RUN go mod tidy +CMD ["gow", "run", "main.go"] diff --git a/README.md b/README.md index 446bd91..f835af3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# akuko-geoparquet-temporal-tooling +# akuko-temporal-go-tooling This Go module provides tools for executing various temporal activities related to GeoParquet files. It includes functionality for converting a GeoParquet file to GeoJSON and sanitizing GeoJSON feature property names. @@ -19,13 +19,13 @@ To build the binary, run the following command: Build ```bash -go build -o ./bin/akuko-geoparquet-temporal-tooling +go build -o ./bin/akuko-temporal-go-tooling ``` Run ```bash -./bin/akuko-geoparquet-temporal-tooling +./bin/akuko-temporal-go-tooling ``` or run the go program directly: diff --git a/activities/geoparquet_to_geojson.go b/activities/geoparquet_to_geojson.go index 698ba9d..df2b29a 100644 --- a/activities/geoparquet_to_geojson.go +++ b/activities/geoparquet_to_geojson.go @@ -8,8 +8,8 @@ import ( "os" "github.com/apache/arrow/go/v14/parquet/file" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geojson" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geoparquet" + "github.com/onaio/akuko-temporal-go-tooling/internal/geojson" + "github.com/onaio/akuko-temporal-go-tooling/internal/geoparquet" ) // ReadFileBytes reads a file from a filepath and returns its contents as a byte slice diff --git a/go.mod b/go.mod index 4a96611..c69004d 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/onaio/akuko-geoparquet-temporal-tooling +module github.com/onaio/akuko-temporal-go-tooling go 1.20 diff --git a/internal/geojson/featurereader.go b/internal/geojson/featurereader.go index c4027ee..808d9e3 100644 --- a/internal/geojson/featurereader.go +++ b/internal/geojson/featurereader.go @@ -8,7 +8,7 @@ import ( "regexp" "strings" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo" + "github.com/onaio/akuko-temporal-go-tooling/internal/geo" "github.com/paulmach/orb" orbjson "github.com/paulmach/orb/geojson" ) diff --git a/internal/geojson/featurereader_test.go b/internal/geojson/featurereader_test.go index 0719b03..5ded0ec 100644 --- a/internal/geojson/featurereader_test.go +++ b/internal/geojson/featurereader_test.go @@ -5,8 +5,8 @@ import ( "os" "testing" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geojson" + "github.com/onaio/akuko-temporal-go-tooling/internal/geo" + "github.com/onaio/akuko-temporal-go-tooling/internal/geojson" "github.com/paulmach/orb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/internal/geojson/geojson.go b/internal/geojson/geojson.go index d5f9220..447ee91 100644 --- a/internal/geojson/geojson.go +++ b/internal/geojson/geojson.go @@ -5,9 +5,9 @@ import ( "io" "github.com/apache/arrow/go/v14/parquet" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geoparquet" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/pqutil" + "github.com/onaio/akuko-temporal-go-tooling/internal/geo" + "github.com/onaio/akuko-temporal-go-tooling/internal/geoparquet" + "github.com/onaio/akuko-temporal-go-tooling/internal/pqutil" ) const primaryColumn = "geometry" diff --git a/internal/geojson/geojson_test.go b/internal/geojson/geojson_test.go index 221ca29..e8b4259 100644 --- a/internal/geojson/geojson_test.go +++ b/internal/geojson/geojson_test.go @@ -28,10 +28,10 @@ import ( "github.com/apache/arrow/go/v14/parquet/file" "github.com/apache/arrow/go/v14/parquet/pqarrow" "github.com/apache/arrow/go/v14/parquet/schema" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geojson" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geoparquet" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/pqutil" + "github.com/onaio/akuko-temporal-go-tooling/internal/geo" + "github.com/onaio/akuko-temporal-go-tooling/internal/geojson" + "github.com/onaio/akuko-temporal-go-tooling/internal/geoparquet" + "github.com/onaio/akuko-temporal-go-tooling/internal/pqutil" "github.com/paulmach/orb" "github.com/paulmach/orb/encoding/wkb" "github.com/stretchr/testify/assert" diff --git a/internal/geojson/recordwriter.go b/internal/geojson/recordwriter.go index 956e120..704afd7 100644 --- a/internal/geojson/recordwriter.go +++ b/internal/geojson/recordwriter.go @@ -6,8 +6,8 @@ import ( "github.com/apache/arrow/go/v14/arrow" "github.com/apache/arrow/go/v14/arrow/array" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geoparquet" + "github.com/onaio/akuko-temporal-go-tooling/internal/geo" + "github.com/onaio/akuko-temporal-go-tooling/internal/geoparquet" orbjson "github.com/paulmach/orb/geojson" ) diff --git a/internal/geoparquet/featurewriter.go b/internal/geoparquet/featurewriter.go index c5eb39a..6c11103 100644 --- a/internal/geoparquet/featurewriter.go +++ b/internal/geoparquet/featurewriter.go @@ -9,7 +9,7 @@ import ( "github.com/apache/arrow/go/v14/arrow/array" "github.com/apache/arrow/go/v14/parquet" "github.com/apache/arrow/go/v14/parquet/pqarrow" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo" + "github.com/onaio/akuko-temporal-go-tooling/internal/geo" "github.com/paulmach/orb" "github.com/paulmach/orb/encoding/wkb" "github.com/paulmach/orb/encoding/wkt" diff --git a/internal/geoparquet/geoparquet.go b/internal/geoparquet/geoparquet.go index a11817b..8d0d6dd 100644 --- a/internal/geoparquet/geoparquet.go +++ b/internal/geoparquet/geoparquet.go @@ -14,8 +14,8 @@ import ( "github.com/apache/arrow/go/v14/parquet/file" "github.com/apache/arrow/go/v14/parquet/pqarrow" "github.com/apache/arrow/go/v14/parquet/schema" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/pqutil" + "github.com/onaio/akuko-temporal-go-tooling/internal/geo" + "github.com/onaio/akuko-temporal-go-tooling/internal/pqutil" "github.com/paulmach/orb/encoding/wkb" "github.com/paulmach/orb/encoding/wkt" ) diff --git a/internal/geoparquet/geoparquet_test.go b/internal/geoparquet/geoparquet_test.go index b81df02..6aa36e0 100644 --- a/internal/geoparquet/geoparquet_test.go +++ b/internal/geoparquet/geoparquet_test.go @@ -24,9 +24,9 @@ import ( "github.com/apache/arrow/go/v14/arrow/memory" "github.com/apache/arrow/go/v14/parquet/file" "github.com/apache/arrow/go/v14/parquet/pqarrow" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geoparquet" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/test" + "github.com/onaio/akuko-temporal-go-tooling/internal/geo" + "github.com/onaio/akuko-temporal-go-tooling/internal/geoparquet" + "github.com/onaio/akuko-temporal-go-tooling/internal/test" "github.com/paulmach/orb" "github.com/paulmach/orb/encoding/wkb" "github.com/stretchr/testify/assert" diff --git a/internal/geoparquet/metadata.go b/internal/geoparquet/metadata.go index 54daa0d..f032215 100644 --- a/internal/geoparquet/metadata.go +++ b/internal/geoparquet/metadata.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/apache/arrow/go/v14/parquet/metadata" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo" + "github.com/onaio/akuko-temporal-go-tooling/internal/geo" ) const ( diff --git a/internal/pqutil/arrow.go b/internal/pqutil/arrow.go index bde3931..1ae10ee 100644 --- a/internal/pqutil/arrow.go +++ b/internal/pqutil/arrow.go @@ -7,7 +7,7 @@ import ( "sort" "github.com/apache/arrow/go/v14/arrow" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo" + "github.com/onaio/akuko-temporal-go-tooling/internal/geo" ) type ArrowSchemaBuilder struct { diff --git a/internal/pqutil/arrow_test.go b/internal/pqutil/arrow_test.go index 1ca438e..6d4709f 100644 --- a/internal/pqutil/arrow_test.go +++ b/internal/pqutil/arrow_test.go @@ -4,8 +4,8 @@ import ( "fmt" "testing" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/pqutil" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/test" + "github.com/onaio/akuko-temporal-go-tooling/internal/pqutil" + "github.com/onaio/akuko-temporal-go-tooling/internal/test" "github.com/stretchr/testify/require" ) diff --git a/internal/pqutil/parquet_test.go b/internal/pqutil/parquet_test.go index e2fc82a..cabbaca 100644 --- a/internal/pqutil/parquet_test.go +++ b/internal/pqutil/parquet_test.go @@ -6,8 +6,8 @@ import ( "github.com/apache/arrow/go/v14/arrow" "github.com/apache/arrow/go/v14/parquet/pqarrow" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/pqutil" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/test" + "github.com/onaio/akuko-temporal-go-tooling/internal/pqutil" + "github.com/onaio/akuko-temporal-go-tooling/internal/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/internal/pqutil/transform_test.go b/internal/pqutil/transform_test.go index 525a7d6..53e3b0c 100644 --- a/internal/pqutil/transform_test.go +++ b/internal/pqutil/transform_test.go @@ -17,8 +17,8 @@ import ( "github.com/apache/arrow/go/v14/parquet/file" "github.com/apache/arrow/go/v14/parquet/pqarrow" "github.com/apache/arrow/go/v14/parquet/schema" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/pqutil" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/test" + "github.com/onaio/akuko-temporal-go-tooling/internal/pqutil" + "github.com/onaio/akuko-temporal-go-tooling/internal/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -173,7 +173,7 @@ func makeOvertureData(t *testing.T) (string, []byte) { } func TestTransformOverture(t *testing.T) { - // minimal reproduction of https://github.com/onaio/akuko-geoparquet-temporal-tooling/issues/102 + // minimal reproduction of https://github.com/onaio/akuko-temporal-go-tooling/issues/102 expected, parquetData := makeOvertureData(t) input := bytes.NewReader(parquetData) diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 90d6a4e..86f32b4 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -8,7 +8,7 @@ import ( "os" "testing" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/storage" + "github.com/onaio/akuko-temporal-go-tooling/internal/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/internal/storage/http_test.go b/internal/storage/http_test.go index e7bade0..ec9d5eb 100644 --- a/internal/storage/http_test.go +++ b/internal/storage/http_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/storage" + "github.com/onaio/akuko-temporal-go-tooling/internal/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 36c79df..76acbf1 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -7,7 +7,7 @@ import ( "net/http/httptest" "testing" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/storage" + "github.com/onaio/akuko-temporal-go-tooling/internal/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/internal/test/test.go b/internal/test/test.go index cca971d..e017881 100644 --- a/internal/test/test.go +++ b/internal/test/test.go @@ -15,8 +15,8 @@ import ( "github.com/apache/arrow/go/v14/parquet/file" "github.com/apache/arrow/go/v14/parquet/pqarrow" "github.com/apache/arrow/go/v14/parquet/schema" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geojson" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/pqutil" + "github.com/onaio/akuko-temporal-go-tooling/internal/geojson" + "github.com/onaio/akuko-temporal-go-tooling/internal/pqutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/internal/validator/rules.go b/internal/validator/rules.go index 8ae9bb0..eb9bc4a 100644 --- a/internal/validator/rules.go +++ b/internal/validator/rules.go @@ -22,8 +22,8 @@ import ( "github.com/apache/arrow/go/v14/parquet" "github.com/apache/arrow/go/v14/parquet/file" "github.com/apache/arrow/go/v14/parquet/schema" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geoparquet" + "github.com/onaio/akuko-temporal-go-tooling/internal/geo" + "github.com/onaio/akuko-temporal-go-tooling/internal/geoparquet" "github.com/paulmach/orb" "github.com/santhosh-tekuri/jsonschema/v5" ) diff --git a/internal/validator/validator.go b/internal/validator/validator.go index 6eb8d83..5b53567 100644 --- a/internal/validator/validator.go +++ b/internal/validator/validator.go @@ -24,8 +24,8 @@ import ( "github.com/apache/arrow/go/v14/arrow/array" "github.com/apache/arrow/go/v14/parquet" "github.com/apache/arrow/go/v14/parquet/file" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geoparquet" + "github.com/onaio/akuko-temporal-go-tooling/internal/geo" + "github.com/onaio/akuko-temporal-go-tooling/internal/geoparquet" "github.com/paulmach/orb" _ "github.com/santhosh-tekuri/jsonschema/v5/httploader" ) diff --git a/internal/validator/validator_test.go b/internal/validator/validator_test.go index c13a143..acb00c5 100644 --- a/internal/validator/validator_test.go +++ b/internal/validator/validator_test.go @@ -29,11 +29,11 @@ import ( "github.com/apache/arrow/go/v14/parquet" "github.com/apache/arrow/go/v14/parquet/file" "github.com/apache/arrow/go/v14/parquet/pqarrow" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geojson" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geoparquet" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/pqutil" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/test" - "github.com/onaio/akuko-geoparquet-temporal-tooling/internal/validator" + "github.com/onaio/akuko-temporal-go-tooling/internal/geojson" + "github.com/onaio/akuko-temporal-go-tooling/internal/geoparquet" + "github.com/onaio/akuko-temporal-go-tooling/internal/pqutil" + "github.com/onaio/akuko-temporal-go-tooling/internal/test" + "github.com/onaio/akuko-temporal-go-tooling/internal/validator" "github.com/paulmach/orb" "github.com/paulmach/orb/encoding/wkb" "github.com/santhosh-tekuri/jsonschema/v5" diff --git a/main.go b/main.go index 63d6256..9c2a4c1 100644 --- a/main.go +++ b/main.go @@ -8,8 +8,8 @@ import ( "go.temporal.io/sdk/client" "go.temporal.io/sdk/worker" - "github.com/onaio/akuko-geoparquet-temporal-tooling/activities" - "github.com/onaio/akuko-geoparquet-temporal-tooling/workflows" + "github.com/onaio/akuko-temporal-go-tooling/activities" + "github.com/onaio/akuko-temporal-go-tooling/workflows" ) type Response struct { diff --git a/workflows/geoparquet.go b/workflows/geoparquet.go index 13663b7..bbceaca 100644 --- a/workflows/geoparquet.go +++ b/workflows/geoparquet.go @@ -5,7 +5,7 @@ import ( "go.temporal.io/sdk/workflow" - "github.com/onaio/akuko-geoparquet-temporal-tooling/activities" + "github.com/onaio/akuko-temporal-go-tooling/activities" ) func ConvertGeoparquetToGeojson(ctx workflow.Context, params *activities.ConvertGeoParquetToGeoJSONActivityParams) (activities.ConvertGeoParquetToGeoJSONActivityReturnType, error) { From 68ea58b2b734dc1b0751ae9a857c5c0df92b3df8 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Mon, 8 Apr 2024 15:29:29 +0300 Subject: [PATCH 02/13] Update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f835af3..b9fb933 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # akuko-temporal-go-tooling -This Go module provides tools for executing various temporal activities related to GeoParquet files. It includes functionality for converting a GeoParquet file to GeoJSON and sanitizing GeoJSON feature property names. +This Go module provides various Akuko temporal activities written using the golang language. At the moment the activities include functionality for converting a GeoParquet file to GeoJSON and sanitizing GeoJSON feature property names. ## Features From 78cb4e581fb299c08ac1cf84cfccb00f359042b5 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Mon, 8 Apr 2024 15:30:44 +0300 Subject: [PATCH 03/13] Create github action --- .github/workflows/docker-image-build.yaml | 107 ++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 .github/workflows/docker-image-build.yaml diff --git a/.github/workflows/docker-image-build.yaml b/.github/workflows/docker-image-build.yaml new file mode 100644 index 0000000..e0551bb --- /dev/null +++ b/.github/workflows/docker-image-build.yaml @@ -0,0 +1,107 @@ +name: AWS ECR docker image build +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + release: + types: + - 'released' + workflow_dispatch: + inputs: + versionTag: + description: "Version Tag" + required: true + default: '' + branchName: + description: "Branch Name" + required: true + default: '' + tagDockerTargetName: + description: "Docker Target" + required: true + default: 'production' + branchDockerTargetName: + description: "Docker Target" + required: true + default: 'dev' +jobs: + main: + runs-on: ubuntu-20.04 + steps: + - name: Get the tag version + id: get-tag-version + if: ${{ github.event.inputs.versionTag == '' && github.event_name == 'release' }} + run: echo ::set-output name=TAG_VERSION::${GITHUB_REF/refs\/tags\//} + + - name: Get Branch name + id: get-branch-name + uses: tj-actions/branch-names@v5.2 + if: | + ${{ github.event.inputs.branchName == '' && github.event_name == 'pull_request' }} || ${{ github.event.inputs.branchName == '' && github.event_name == 'push' }} + + - name: Checkout to tag version + uses: actions/checkout@v2 + if: ${{ github.event.inputs.versionTag == '' && github.event_name == 'release' }} + with: + ref: ${{ github.event.inputs.versionTag || steps.get-tag-version.outputs.TAG_VERSION }} + + - name: Checkout to branch + uses: actions/checkout@v2 + if: | + ${{ github.event.inputs.branchName == '' && github.event_name == 'pull_request' }} || ${{ github.event.inputs.branchName == '' && github.event_name == 'push' }} + with: + ref: ${{ github.event.inputs.branchName || steps.get-branch-name.outputs.current_branch }} + + - name: Install dependencies + run: yarn + + - name: Run tests + run: yarn test + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-central-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build and push a tag + id: docker-build-tag + uses: docker/build-push-action@v2 + if: ${{ github.event.inputs.versionTag == '' && github.event_name == 'release' }} + with: + context: . + target: ${{ github.event.inputs.tagDockerTargetName }} + file: Dockerfile + platforms: linux/amd64 + push: true + tags: | + ${{ steps.login-ecr.outputs.registry }}/onaio/akuko-temporal-go-tooling:${{ github.event.inputs.versionTag || steps.get-tag-version.outputs.TAG_VERSION }} + + - name: Build and push a branch + id: docker-build-branch + uses: docker/build-push-action@v2 + if: | + ${{ github.event.inputs.branchName == '' && github.event_name == 'pull_request' }} || ${{ github.event.inputs.branchName == '' && github.event_name == 'push' }} + with: + context: . + target: ${{ github.event.inputs.branchDockerTargetName }} + file: Dockerfile + platforms: linux/amd64 + push: true + tags: | + ${{ steps.login-ecr.outputs.registry }}/onaio/akuko-temporal-go-tooling:${{ github.event.inputs.branchName || steps.get-branch-name.outputs.current_branch }} + + - name: Image digest tag + if: ${{ github.event.inputs.versionTag == '' && github.event_name == 'release' }} + run: echo ${{ steps.docker-build-tag.outputs.digest }} + + - name: Image digest branch + if: | + ${{ github.event.inputs.branchName == '' && github.event_name == 'pull_request' }} || ${{ github.event.inputs.branchName == '' && github.event_name == 'push' }} + run: echo ${{ steps.docker-build-branch.outputs.digest }} \ No newline at end of file From 76b88d589a423743f0a3b9d9fe19d388841db9e3 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Mon, 8 Apr 2024 15:41:43 +0300 Subject: [PATCH 04/13] Remove yarn steps --- .github/workflows/docker-image-build.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/docker-image-build.yaml b/.github/workflows/docker-image-build.yaml index e0551bb..cf4cfae 100644 --- a/.github/workflows/docker-image-build.yaml +++ b/.github/workflows/docker-image-build.yaml @@ -53,12 +53,6 @@ jobs: with: ref: ${{ github.event.inputs.branchName || steps.get-branch-name.outputs.current_branch }} - - name: Install dependencies - run: yarn - - - name: Run tests - run: yarn test - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: From fb5f8f52e4aff9d9ccbd40c207c685553b2f67c1 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Mon, 8 Apr 2024 17:30:54 +0300 Subject: [PATCH 05/13] Update health check error logic --- healthcheck/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/healthcheck/main.go b/healthcheck/main.go index 553af1e..699d2d7 100644 --- a/healthcheck/main.go +++ b/healthcheck/main.go @@ -14,7 +14,7 @@ func main() { } temporalClient, err := client.Dial(clientOptions) if err != nil { - fmt.Println("Unable to create a Temporal Client: %s", err) + panic(err) } fmt.Println("Health check passed.") defer temporalClient.Close() From 147c7064c1713264618780fa5abd8494130738d4 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Mon, 8 Apr 2024 17:41:51 +0300 Subject: [PATCH 06/13] Switch from gow to go --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f2aef3f..aa3fd11 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,4 +10,4 @@ ENTRYPOINT ["./bin/akuko-temporal-go-tooling"] FROM base as dev RUN go mod tidy -CMD ["gow", "run", "main.go"] +CMD ["go", "run", "main.go"] From fc673c7bb99fb27284c012e41a5e6ef207e6f0b3 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Mon, 8 Apr 2024 20:55:34 +0300 Subject: [PATCH 07/13] Fix namespace env var --- healthcheck/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/healthcheck/main.go b/healthcheck/main.go index 699d2d7..e0bedb4 100644 --- a/healthcheck/main.go +++ b/healthcheck/main.go @@ -10,7 +10,7 @@ import ( func main() { clientOptions := client.Options{ HostPort: os.Getenv("TEMPORAL_HOST"), - Namespace: os.Getenv("SOURCE_CREATION_AND_UPDATING_TEMPORAL_NAMESPACE"), + Namespace: os.Getenv("TEMPORAL_NAMESPACE"), } temporalClient, err := client.Dial(clientOptions) if err != nil { From 47e95fad77ddbc32aca242196a598fe24e5bee59 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Tue, 9 Apr 2024 10:09:53 +0300 Subject: [PATCH 08/13] Update dockerfile --- Dockerfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index aa3fd11..a96c0c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,14 @@ COPY . . FROM base as production RUN go build -o ./bin/akuko-temporal-go-tooling +WORKDIR /app/healthcheck +RUN go build -o ../bin/healthcheck +WORKDIR /app ENTRYPOINT ["./bin/akuko-temporal-go-tooling"] FROM base as dev -RUN go mod tidy -CMD ["go", "run", "main.go"] +RUN go build -o ./bin/akuko-temporal-go-tooling +WORKDIR /app/healthcheck +RUN go build -o ../bin/healthcheck +WORKDIR /app +ENTRYPOINT ["./bin/akuko-temporal-go-tooling"] From a717647196d52df4020bb5fc00c319b47389807b Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Tue, 9 Apr 2024 10:15:05 +0300 Subject: [PATCH 09/13] Update root path to /srv/app --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index a96c0c1..b3c929a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,19 @@ FROM golang:1.20 AS base -WORKDIR /app +WORKDIR /srv/app COPY . . FROM base as production RUN go build -o ./bin/akuko-temporal-go-tooling -WORKDIR /app/healthcheck +WORKDIR /srv/app/healthcheck RUN go build -o ../bin/healthcheck -WORKDIR /app +WORKDIR /srv/app ENTRYPOINT ["./bin/akuko-temporal-go-tooling"] FROM base as dev RUN go build -o ./bin/akuko-temporal-go-tooling -WORKDIR /app/healthcheck +WORKDIR /srv/app/healthcheck RUN go build -o ../bin/healthcheck -WORKDIR /app +WORKDIR /srv/app ENTRYPOINT ["./bin/akuko-temporal-go-tooling"] From 18474be4ad30acb6fcb941085e8f54b0e9f0f891 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Tue, 9 Apr 2024 20:50:35 +0300 Subject: [PATCH 10/13] Set target to default --- .github/workflows/docker-image-build.yaml | 4 ++-- Dockerfile | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-image-build.yaml b/.github/workflows/docker-image-build.yaml index cf4cfae..0418ba9 100644 --- a/.github/workflows/docker-image-build.yaml +++ b/.github/workflows/docker-image-build.yaml @@ -20,11 +20,11 @@ on: tagDockerTargetName: description: "Docker Target" required: true - default: 'production' + default: 'default' branchDockerTargetName: description: "Docker Target" required: true - default: 'dev' + default: 'default' jobs: main: runs-on: ubuntu-20.04 diff --git a/Dockerfile b/Dockerfile index b3c929a..dfd1a20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,14 +4,7 @@ WORKDIR /srv/app COPY . . -FROM base as production -RUN go build -o ./bin/akuko-temporal-go-tooling -WORKDIR /srv/app/healthcheck -RUN go build -o ../bin/healthcheck -WORKDIR /srv/app -ENTRYPOINT ["./bin/akuko-temporal-go-tooling"] - -FROM base as dev +FROM base as default RUN go build -o ./bin/akuko-temporal-go-tooling WORKDIR /srv/app/healthcheck RUN go build -o ../bin/healthcheck From e1be08df0dadf053d5edf3e8659eafa443c8a682 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Thu, 11 Apr 2024 17:31:16 +0300 Subject: [PATCH 11/13] Refactor feature id --- .../sanitize_geojson_feature_properties.go | 15 ++++--- internal/geojson/featurereader.go | 39 ------------------- 2 files changed, 7 insertions(+), 47 deletions(-) diff --git a/activities/sanitize_geojson_feature_properties.go b/activities/sanitize_geojson_feature_properties.go index 1d66a6e..fe304db 100644 --- a/activities/sanitize_geojson_feature_properties.go +++ b/activities/sanitize_geojson_feature_properties.go @@ -24,18 +24,13 @@ func sanitizeStringToCubeSyntax(str string) string { } // sanitizeProperties sanitizes the keys in the properties map to conform to Cube syntax rules. -// It also adds a feature_id key for each property with value index + 1. -func sanitizeProperties(properties map[string]interface{}) map[string]interface{} { +func sanitizeProperties(properties map[string]interface{}, feature_id int) map[string]interface{} { sanitizedProperties := make(map[string]interface{}) - index := 0 for key, value := range properties { sanitizedKey := sanitizeStringToCubeSyntax(key) sanitizedProperties[sanitizedKey] = value - index++ - - // Add feature_id key for each property - sanitizedProperties["feature_id"] = index + sanitizedProperties["feature_id"] = feature_id } return sanitizedProperties @@ -61,8 +56,10 @@ func SanitizeGeoJSONFile(inputFile string, outputFile string) error { return fmt.Errorf("no features found in GeoJSON data") } + feature_id := 0 // Sanitize feature properties for _, feature := range features { + feature_id++ featureMap, ok := feature.(map[string]interface{}) if !ok { return fmt.Errorf("invalid feature format") @@ -71,7 +68,9 @@ func SanitizeGeoJSONFile(inputFile string, outputFile string) error { if !ok { return fmt.Errorf("no properties found in feature") } - featureMap["properties"] = sanitizeProperties(properties) + // sanitizeProperties sanitizes the keys in the properties map to conform to Cube syntax rules. + // It also adds a feature_id key for each feature with value index + 1. + featureMap["properties"] = sanitizeProperties(properties, feature_id) } // Marshal sanitized GeoJSON data diff --git a/internal/geojson/featurereader.go b/internal/geojson/featurereader.go index 808d9e3..70a52b0 100644 --- a/internal/geojson/featurereader.go +++ b/internal/geojson/featurereader.go @@ -5,8 +5,6 @@ import ( "errors" "fmt" "io" - "regexp" - "strings" "github.com/onaio/akuko-temporal-go-tooling/internal/geo" "github.com/paulmach/orb" @@ -105,8 +103,6 @@ func (r *FeatureReader) Read() (*geo.Feature, error) { return nil, fmt.Errorf("trouble parsing properties: %w", err) } feature.Properties = properties - fmt.Println("feature.Properties: %s", feature.Properties) - fmt.Println("properties: %s", properties) continue } @@ -253,10 +249,6 @@ func (r *FeatureReader) readFeature() (*geo.Feature, error) { if err := r.decoder.Decode(feature); err != nil { return nil, err } - // Sanitize feature properties keys - fmt.Println("Unsanitized Properties: %s", feature.Properties) - feature.Properties = sanitizeProperties(feature.Properties) - fmt.Println("Sanitizes Properties: %s", feature.Properties) return feature, nil } @@ -279,34 +271,3 @@ func (r *FeatureReader) readGeometryCollection() (*geo.Feature, error) { feature.Geometry = orb.Collection(geometries) return feature, nil } - -// sanitizeProperties sanitizes the keys in the properties map to conform to Cube syntax rules. -func sanitizeProperties(properties map[string]interface{}) map[string]interface{} { - sanitizedProperties := make(map[string]interface{}) - - index := 0 - for key, value := range properties { - sanitizedKey := sanitizeStringToCubeSyntax(key) - sanitizedProperties[sanitizedKey] = value - index++ - - // Add feature_id key for each property - sanitizedProperties["feature_id"] = index - } - - return sanitizedProperties -} - -// sanitizeStringToCubeSyntax sanitizes a string to conform to Cube syntax rules. -func sanitizeStringToCubeSyntax(str string) string { - // Cube doesn't allow numbers as the first character. - firstCharacter := str[:1] - if strings.Contains("0123456789", firstCharacter) { - str = "_" + str - } - - // Replace non-alphanumeric characters with underscore and convert to lowercase. - reg := regexp.MustCompile("[^A-Za-z0-9]+") - sanitized := reg.ReplaceAllString(str, "_") - return strings.ToLower(sanitized) -} From 5c25fc02149bde206b9c8b39c79f75028011da3b Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Fri, 12 Apr 2024 11:10:29 +0300 Subject: [PATCH 12/13] Heartbeat activities --- activities/geoparquet_to_geojson.go | 13 +++++++++++++ activities/sanitize_geojson_feature_properties.go | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/activities/geoparquet_to_geojson.go b/activities/geoparquet_to_geojson.go index df2b29a..091298c 100644 --- a/activities/geoparquet_to_geojson.go +++ b/activities/geoparquet_to_geojson.go @@ -10,6 +10,7 @@ import ( "github.com/apache/arrow/go/v14/parquet/file" "github.com/onaio/akuko-temporal-go-tooling/internal/geojson" "github.com/onaio/akuko-temporal-go-tooling/internal/geoparquet" + "go.temporal.io/sdk/activity" ) // ReadFileBytes reads a file from a filepath and returns its contents as a byte slice @@ -53,12 +54,18 @@ type ConvertGeoParquetToGeoJSONActivityReturnType struct { } func ConvertGeoParquetToGeoJSONActivity(ctx context.Context, params *ConvertGeoParquetToGeoJSONActivityParams) (*ConvertGeoParquetToGeoJSONActivityReturnType, error) { + message := "Reading geoparquet file bytes" + fmt.Println(message) + activity.RecordHeartbeat(ctx, message) fileBytes, err := ReadFileBytes(params.GeoParquetFilePath) if err != nil { fmt.Println("Error:", err) return nil, err } + message = "Converting geoparquet to geojson" + fmt.Println(message) + activity.RecordHeartbeat(ctx, message) // Convert from GeoParquet to GeoJSON geoJSONBuffer := &bytes.Buffer{} err = geojson.FromParquet(bytes.NewReader(fileBytes), geoJSONBuffer) @@ -74,6 +81,9 @@ func ConvertGeoParquetToGeoJSONActivity(ctx context.Context, params *ConvertGeoP } defer reader.Close() + message = "Getting geoparquet metadata" + fmt.Println(message) + activity.RecordHeartbeat(ctx, message) metadata, metadataErr := geoparquet.GetMetadata(reader.MetaData().GetKeyValueMetadata()) if metadataErr != nil { fmt.Println("Error converting from Parquet to GeoJSON: %v", metadataErr) @@ -82,6 +92,9 @@ func ConvertGeoParquetToGeoJSONActivity(ctx context.Context, params *ConvertGeoP fmt.Println("MetaData: %s", &metadata) + message = "Writting geojson data to disk" + fmt.Println(message) + activity.RecordHeartbeat(ctx, message) err = WriteFileBytes(params.GeoJSONFilePath, geoJSONBuffer.Bytes()) if err != nil { fmt.Println("Error writing file to disk: %v", err) diff --git a/activities/sanitize_geojson_feature_properties.go b/activities/sanitize_geojson_feature_properties.go index fe304db..0c0c9ee 100644 --- a/activities/sanitize_geojson_feature_properties.go +++ b/activities/sanitize_geojson_feature_properties.go @@ -7,6 +7,8 @@ import ( "io/ioutil" "regexp" "strings" + + "go.temporal.io/sdk/activity" ) // sanitizeStringToCubeSyntax sanitizes a string to conform to Cube syntax rules. @@ -96,6 +98,8 @@ type SanitizeGeoJSONFeaturePropertiesActivityReturnType struct { } func SanitizeGeoJSONFeaturePropertiesActivity(ctx context.Context, params *SanitizeGeoJSONFeaturePropertiesActivityParams) (*SanitizeGeoJSONFeaturePropertiesActivityReturnType, error) { + message := "Sanitizing geojson file" + activity.RecordHeartbeat(ctx, message) inputFile := params.GeoJSONFilePath outputFile := params.GeoJSONFilePath @@ -105,6 +109,8 @@ func SanitizeGeoJSONFeaturePropertiesActivity(ctx context.Context, params *Sanit return nil, err } + message = "GeoJSON file sanitized successfully." + activity.RecordHeartbeat(ctx, message) fmt.Println("GeoJSON file sanitized successfully.") var data = SanitizeGeoJSONFeaturePropertiesActivityReturnType{ FilePath: outputFile, From 968881d481dc6ac0cd92d07fcbf5f53a4772aafd Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Fri, 12 Apr 2024 18:30:09 +0300 Subject: [PATCH 13/13] Fix linting issue on println statement --- activities/geoparquet_to_geojson.go | 10 +++++----- main.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/activities/geoparquet_to_geojson.go b/activities/geoparquet_to_geojson.go index 091298c..2caa6cb 100644 --- a/activities/geoparquet_to_geojson.go +++ b/activities/geoparquet_to_geojson.go @@ -70,13 +70,13 @@ func ConvertGeoParquetToGeoJSONActivity(ctx context.Context, params *ConvertGeoP geoJSONBuffer := &bytes.Buffer{} err = geojson.FromParquet(bytes.NewReader(fileBytes), geoJSONBuffer) if err != nil { - fmt.Println("Error converting from Parquet to GeoJSON: %v", err) + fmt.Println("Error converting from Parquet to GeoJSON: ", err) return nil, err } reader, readerErr := newFileReader(params.GeoParquetFilePath) if readerErr != nil { - fmt.Println("Error converting from Parquet to GeoJSON: %v", readerErr) + fmt.Println("Error converting from Parquet to GeoJSON: ", readerErr) return nil, readerErr } defer reader.Close() @@ -86,18 +86,18 @@ func ConvertGeoParquetToGeoJSONActivity(ctx context.Context, params *ConvertGeoP activity.RecordHeartbeat(ctx, message) metadata, metadataErr := geoparquet.GetMetadata(reader.MetaData().GetKeyValueMetadata()) if metadataErr != nil { - fmt.Println("Error converting from Parquet to GeoJSON: %v", metadataErr) + fmt.Println("Error converting from Parquet to GeoJSON: ", metadataErr) return nil, metadataErr } - fmt.Println("MetaData: %s", &metadata) + fmt.Println("MetaData: ", &metadata) message = "Writting geojson data to disk" fmt.Println(message) activity.RecordHeartbeat(ctx, message) err = WriteFileBytes(params.GeoJSONFilePath, geoJSONBuffer.Bytes()) if err != nil { - fmt.Println("Error writing file to disk: %v", err) + fmt.Println("Error writing file to disk: ", err) return nil, err } var data = ConvertGeoParquetToGeoJSONActivityReturnType{ diff --git a/main.go b/main.go index 9c2a4c1..c35f358 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,7 @@ func main() { } temporalClient, err := client.Dial(clientOptions) if err != nil { - fmt.Println("Unable to create a Temporal Client: %s", err) + fmt.Println("Unable to create a Temporal Client: ", err) } if temporalClient != nil { defer temporalClient.Close()