Maintenance Task check list

.github/workflows/docker-image-build.yaml

@@ -0,0 +1,101 @@
+name: AWS ECR docker image build
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  release:
+    types:
+      - 'released'
+  workflow_dispatch:
+    inputs:
+      versionTag:
+        description: "Version Tag"
+        required: true
+        default: ''
+      branchName:
+        description: "Branch Name"
+        required: true
+        default: ''
+      tagDockerTargetName:
+        description: "Docker Target"
+        required: true
+        default: 'default'
+      branchDockerTargetName:
+        description: "Docker Target"
+        required: true
+        default: 'default'
+jobs:
+  main:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Get the tag version
+        id: get-tag-version
+        if: ${{ github.event.inputs.versionTag == '' && github.event_name == 'release' }}
+        run: echo ::set-output name=TAG_VERSION::${GITHUB_REF/refs\/tags\//}
+
+      - name: Get Branch name
+        id: get-branch-name
+        uses: tj-actions/branch-names@v5.2
+        if: |
+          ${{ github.event.inputs.branchName == '' && github.event_name == 'pull_request' }} || ${{ github.event.inputs.branchName == '' && github.event_name == 'push' }}
+
+      - name: Checkout to tag version
+        uses: actions/checkout@v2
+        if: ${{ github.event.inputs.versionTag == '' && github.event_name == 'release' }}
+        with:
+          ref: ${{ github.event.inputs.versionTag || steps.get-tag-version.outputs.TAG_VERSION }}
+
+      - name: Checkout to branch
+        uses: actions/checkout@v2
+        if: |
+          ${{ github.event.inputs.branchName == '' && github.event_name == 'pull_request' }} || ${{ github.event.inputs.branchName == '' && github.event_name == 'push' }}
+        with:
+          ref: ${{ github.event.inputs.branchName || steps.get-branch-name.outputs.current_branch }}
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-central-1
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Build and push a tag
+        id: docker-build-tag
+        uses: docker/build-push-action@v2
+        if: ${{ github.event.inputs.versionTag == '' && github.event_name == 'release' }}
+        with:
+          context: .
+          target: ${{ github.event.inputs.tagDockerTargetName }}
+          file: Dockerfile
+          platforms: linux/amd64
+          push: true
+          tags: |
+            ${{ steps.login-ecr.outputs.registry }}/onaio/akuko-temporal-go-tooling:${{ github.event.inputs.versionTag || steps.get-tag-version.outputs.TAG_VERSION }}
+
+      - name: Build and push a branch
+        id: docker-build-branch
+        uses: docker/build-push-action@v2
+        if: |
+          ${{ github.event.inputs.branchName == '' && github.event_name == 'pull_request' }} || ${{ github.event.inputs.branchName == '' && github.event_name == 'push' }}
+        with:
+          context: .
+          target: ${{ github.event.inputs.branchDockerTargetName }}
+          file: Dockerfile
+          platforms: linux/amd64
+          push: true
+          tags: |
+            ${{ steps.login-ecr.outputs.registry }}/onaio/akuko-temporal-go-tooling:${{ github.event.inputs.branchName || steps.get-branch-name.outputs.current_branch }}
+
+      - name: Image digest tag
+        if: ${{ github.event.inputs.versionTag == '' && github.event_name == 'release' }}
+        run: echo ${{ steps.docker-build-tag.outputs.digest }}
+
+      - name: Image digest branch
+        if: |
+          ${{ github.event.inputs.branchName == '' && github.event_name == 'pull_request' }} || ${{ github.event.inputs.branchName == '' && github.event_name == 'push' }}
+        run: echo ${{ steps.docker-build-branch.outputs.digest }}

Dockerfile

@@ -1,9 +1,12 @@
-FROM golang:1.20 AS builder
+FROM golang:1.20 AS base
 
-WORKDIR /app
+WORKDIR /srv/app
 
 COPY . .
 
-RUN go build -o ./bin/akuko-geoparquet-temporal-tooling
-
-ENTRYPOINT ["./bin/akuko-geoparquet-temporal-tooling"]
+FROM base as default
+RUN go build -o ./bin/akuko-temporal-go-tooling
+WORKDIR /srv/app/healthcheck
+RUN go build -o ../bin/healthcheck
+WORKDIR /srv/app
+ENTRYPOINT ["./bin/akuko-temporal-go-tooling"]

README.md

@@ -1,6 +1,6 @@
-# akuko-geoparquet-temporal-tooling
+# akuko-temporal-go-tooling
 
-This Go module provides tools for executing various temporal activities related to GeoParquet files. It includes functionality for converting a GeoParquet file to GeoJSON and sanitizing GeoJSON feature property names.
+This Go module provides various Akuko temporal activities written using the golang language. At the moment the activities include functionality for converting a GeoParquet file to GeoJSON and sanitizing GeoJSON feature property names.
 
 ## Features
 
@@ -19,13 +19,13 @@ To build the binary, run the following command:
 Build
 
 ```bash
-go build -o ./bin/akuko-geoparquet-temporal-tooling
+go build -o ./bin/akuko-temporal-go-tooling
 ```
 
 Run
 
 ```bash
-./bin/akuko-geoparquet-temporal-tooling
+./bin/akuko-temporal-go-tooling
 ```
 or run the go program directly:
 

activities/geoparquet_to_geojson.go

@@ -8,8 +8,9 @@ import (
 	"os"
 
 	"github.com/apache/arrow/go/v14/parquet/file"
-	"github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geojson"
-	"github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geoparquet"
+	"github.com/onaio/akuko-temporal-go-tooling/internal/geojson"
+	"github.com/onaio/akuko-temporal-go-tooling/internal/geoparquet"
+	"go.temporal.io/sdk/activity"
 )
 
 // ReadFileBytes reads a file from a filepath and returns its contents as a byte slice
@@ -53,38 +54,50 @@ type ConvertGeoParquetToGeoJSONActivityReturnType struct {
 }
 
 func ConvertGeoParquetToGeoJSONActivity(ctx context.Context, params *ConvertGeoParquetToGeoJSONActivityParams) (*ConvertGeoParquetToGeoJSONActivityReturnType, error) {
+	message := "Reading geoparquet file bytes"
+	fmt.Println(message)
+	activity.RecordHeartbeat(ctx, message)
 	fileBytes, err := ReadFileBytes(params.GeoParquetFilePath)
 	if err != nil {
 		fmt.Println("Error:", err)
 		return nil, err
 	}
 
+	message = "Converting geoparquet to geojson"
+	fmt.Println(message)
+	activity.RecordHeartbeat(ctx, message)
 	// Convert from GeoParquet to GeoJSON
 	geoJSONBuffer := &bytes.Buffer{}
 	err = geojson.FromParquet(bytes.NewReader(fileBytes), geoJSONBuffer)
 	if err != nil {
-		fmt.Println("Error converting from Parquet to GeoJSON: %v", err)
+		fmt.Println("Error converting from Parquet to GeoJSON: ", err)
 		return nil, err
 	}
 
 	reader, readerErr := newFileReader(params.GeoParquetFilePath)
 	if readerErr != nil {
-		fmt.Println("Error converting from Parquet to GeoJSON: %v", readerErr)
+		fmt.Println("Error converting from Parquet to GeoJSON: ", readerErr)
 		return nil, readerErr
 	}
 	defer reader.Close()
 
+	message = "Getting geoparquet metadata"
+	fmt.Println(message)
+	activity.RecordHeartbeat(ctx, message)
 	metadata, metadataErr := geoparquet.GetMetadata(reader.MetaData().GetKeyValueMetadata())
 	if metadataErr != nil {
-		fmt.Println("Error converting from Parquet to GeoJSON: %v", metadataErr)
+		fmt.Println("Error converting from Parquet to GeoJSON: ", metadataErr)
 		return nil, metadataErr
 	}
 
-	fmt.Println("MetaData: %s", &metadata)
+	fmt.Println("MetaData: ", &metadata)
 
+	message = "Writting geojson data to disk"
+	fmt.Println(message)
+	activity.RecordHeartbeat(ctx, message)
 	err = WriteFileBytes(params.GeoJSONFilePath, geoJSONBuffer.Bytes())
 	if err != nil {
-		fmt.Println("Error writing file to disk: %v", err)
+		fmt.Println("Error writing file to disk: ", err)
 		return nil, err
 	}
 	var data = ConvertGeoParquetToGeoJSONActivityReturnType{

activities/sanitize_geojson_feature_properties.go

@@ -7,6 +7,8 @@ import (
 	"io/ioutil"
 	"regexp"
 	"strings"
+
+	"go.temporal.io/sdk/activity"
 )
 
 // sanitizeStringToCubeSyntax sanitizes a string to conform to Cube syntax rules.
@@ -24,18 +26,13 @@ func sanitizeStringToCubeSyntax(str string) string {
 }
 
 // sanitizeProperties sanitizes the keys in the properties map to conform to Cube syntax rules.
-// It also adds a feature_id key for each property with value index + 1.
-func sanitizeProperties(properties map[string]interface{}) map[string]interface{} {
+func sanitizeProperties(properties map[string]interface{}, feature_id int) map[string]interface{} {
 	sanitizedProperties := make(map[string]interface{})
 
-	index := 0
 	for key, value := range properties {
 		sanitizedKey := sanitizeStringToCubeSyntax(key)
 		sanitizedProperties[sanitizedKey] = value
-		index++
-
-		// Add feature_id key for each property
-		sanitizedProperties["feature_id"] = index
+		sanitizedProperties["feature_id"] = feature_id
 	}
 
 	return sanitizedProperties
@@ -61,8 +58,10 @@ func SanitizeGeoJSONFile(inputFile string, outputFile string) error {
 		return fmt.Errorf("no features found in GeoJSON data")
 	}
 
+	feature_id := 0
 	// Sanitize feature properties
 	for _, feature := range features {
+		feature_id++
 		featureMap, ok := feature.(map[string]interface{})
 		if !ok {
 			return fmt.Errorf("invalid feature format")
@@ -71,7 +70,9 @@ func SanitizeGeoJSONFile(inputFile string, outputFile string) error {
 		if !ok {
 			return fmt.Errorf("no properties found in feature")
 		}
-		featureMap["properties"] = sanitizeProperties(properties)
+		// sanitizeProperties sanitizes the keys in the properties map to conform to Cube syntax rules.
+		// It also adds a feature_id key for each feature with value index + 1.
+		featureMap["properties"] = sanitizeProperties(properties, feature_id)
 	}
 
 	// Marshal sanitized GeoJSON data
@@ -97,6 +98,8 @@ type SanitizeGeoJSONFeaturePropertiesActivityReturnType struct {
 }
 
 func SanitizeGeoJSONFeaturePropertiesActivity(ctx context.Context, params *SanitizeGeoJSONFeaturePropertiesActivityParams) (*SanitizeGeoJSONFeaturePropertiesActivityReturnType, error) {
+	message := "Sanitizing geojson file"
+	activity.RecordHeartbeat(ctx, message)
 	inputFile := params.GeoJSONFilePath
 	outputFile := params.GeoJSONFilePath
 
@@ -106,6 +109,8 @@ func SanitizeGeoJSONFeaturePropertiesActivity(ctx context.Context, params *Sanit
 		return nil, err
 	}
 
+	message = "GeoJSON file sanitized successfully."
+	activity.RecordHeartbeat(ctx, message)
 	fmt.Println("GeoJSON file sanitized successfully.")
 	var data = SanitizeGeoJSONFeaturePropertiesActivityReturnType{
 		FilePath: outputFile,

go.mod

@@ -1,4 +1,4 @@
-module github.com/onaio/akuko-geoparquet-temporal-tooling
+module github.com/onaio/akuko-temporal-go-tooling
 
 go 1.20
 

healthcheck/main.go

@@ -10,11 +10,11 @@ import (
 func main() {
 	clientOptions := client.Options{
 		HostPort:  os.Getenv("TEMPORAL_HOST"),
-		Namespace: os.Getenv("SOURCE_CREATION_AND_UPDATING_TEMPORAL_NAMESPACE"),
+		Namespace: os.Getenv("TEMPORAL_NAMESPACE"),
 	}
 	temporalClient, err := client.Dial(clientOptions)
 	if err != nil {
-		fmt.Println("Unable to create a Temporal Client: %s", err)
+		panic(err)
 	}
 	fmt.Println("Health check passed.")
 	defer temporalClient.Close()

internal/geojson/featurereader.go

@@ -5,10 +5,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"regexp"
-	"strings"
 
-	"github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo"
+	"github.com/onaio/akuko-temporal-go-tooling/internal/geo"
 	"github.com/paulmach/orb"
 	orbjson "github.com/paulmach/orb/geojson"
 )
@@ -105,8 +103,6 @@ func (r *FeatureReader) Read() (*geo.Feature, error) {
 				return nil, fmt.Errorf("trouble parsing properties: %w", err)
 			}
 			feature.Properties = properties
-			fmt.Println("feature.Properties: %s", feature.Properties)
-			fmt.Println("properties: %s", properties)
 			continue
 		}
 
@@ -253,10 +249,6 @@ func (r *FeatureReader) readFeature() (*geo.Feature, error) {
 	if err := r.decoder.Decode(feature); err != nil {
 		return nil, err
 	}
-	// Sanitize feature properties keys
-	fmt.Println("Unsanitized Properties: %s", feature.Properties)
-	feature.Properties = sanitizeProperties(feature.Properties)
-	fmt.Println("Sanitizes Properties: %s", feature.Properties)
 	return feature, nil
 }
 
@@ -279,34 +271,3 @@ func (r *FeatureReader) readGeometryCollection() (*geo.Feature, error) {
 	feature.Geometry = orb.Collection(geometries)
 	return feature, nil
 }
-
-// sanitizeProperties sanitizes the keys in the properties map to conform to Cube syntax rules.
-func sanitizeProperties(properties map[string]interface{}) map[string]interface{} {
-	sanitizedProperties := make(map[string]interface{})
-
-	index := 0
-	for key, value := range properties {
-		sanitizedKey := sanitizeStringToCubeSyntax(key)
-		sanitizedProperties[sanitizedKey] = value
-		index++
-
-		// Add feature_id key for each property
-		sanitizedProperties["feature_id"] = index
-	}
-
-	return sanitizedProperties
-}
-
-// sanitizeStringToCubeSyntax sanitizes a string to conform to Cube syntax rules.
-func sanitizeStringToCubeSyntax(str string) string {
-	// Cube doesn't allow numbers as the first character.
-	firstCharacter := str[:1]
-	if strings.Contains("0123456789", firstCharacter) {
-		str = "_" + str
-	}
-
-	// Replace non-alphanumeric characters with underscore and convert to lowercase.
-	reg := regexp.MustCompile("[^A-Za-z0-9]+")
-	sanitized := reg.ReplaceAllString(str, "_")
-	return strings.ToLower(sanitized)
-}

internal/geojson/featurereader_test.go

@@ -5,8 +5,8 @@ import (
 	"os"
 	"testing"
 
-	"github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo"
-	"github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geojson"
+	"github.com/onaio/akuko-temporal-go-tooling/internal/geo"
+	"github.com/onaio/akuko-temporal-go-tooling/internal/geojson"
 	"github.com/paulmach/orb"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"

internal/geojson/geojson.go

@@ -5,9 +5,9 @@ import (
 	"io"
 
 	"github.com/apache/arrow/go/v14/parquet"
-	"github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geo"
-	"github.com/onaio/akuko-geoparquet-temporal-tooling/internal/geoparquet"
-	"github.com/onaio/akuko-geoparquet-temporal-tooling/internal/pqutil"
+	"github.com/onaio/akuko-temporal-go-tooling/internal/geo"
+	"github.com/onaio/akuko-temporal-go-tooling/internal/geoparquet"
+	"github.com/onaio/akuko-temporal-go-tooling/internal/pqutil"
 )
 
 const primaryColumn = "geometry"