Merge pull request #20 from austin1237/jobNotifier

Job Api

.github/workflows/pr.yml

@@ -41,6 +41,14 @@ jobs:
       run: go build -v ./...
       working-directory: proxy
 
+    - name: Build jobNotifier
+      run: go build -v ./...
+      working-directory: jobNotifier
+
+    - name: Test jobNotifier
+      run: go test -v ./...
+      working-directory: jobNotifier
+
     - name: Setup Node.js
       uses: actions/setup-node@v4
       with:

Makefile

@@ -1,12 +1,15 @@
 # AWS go lambdas running on provided.al2 runtime have to be called bootstrap 
 # https://docs.aws.amazon.com/lambda/latest/dg/golang-handler.html#golang-handler-naming
 
-packageLambdas: packageScraper packageProxy
+packageLambdas: packageScraper packageProxy packageJobNotifier
 
 packageScraper:
 	cd scraper && GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -tags lambda.norpc -o bootstrap main.go && zip bootstrap.zip bootstrap && rm bootstrap && ls
 
 packageProxy:
 	cd proxy && GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -tags lambda.norpc -o bootstrap main.go && zip bootstrap.zip bootstrap && rm bootstrap && ls
 
+packageJobNotifier:
+	cd jobNotifier && GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -tags lambda.norpc -o bootstrap main.go && zip bootstrap.zip bootstrap && rm bootstrap && ls
+
 

go.work

@@ -3,4 +3,5 @@ go 1.21.6
 use (
 	./proxy
 	./scraper
+	./jobNotifier
 )

go.work.sum

@@ -1,2 +1,4 @@
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=

job-openapi.json

@@ -0,0 +1,26 @@
+{
+    "openapi": "3.0.1",
+    "info": {
+      "title": "Job API",
+      "version": "1.0"
+    },
+    "paths" : {
+        "/job" : {
+          "post" : {
+            "responses" : {
+              "default" : {
+                "description" : "Default response for POST /job"
+              }
+            },
+            "x-amazon-apigateway-integration" : {
+              "payloadFormatVersion" : "2.0",
+              "type" : "aws_proxy",
+              "httpMethod" : "POST",
+              "uri" : "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:{account-id}:function:jobNotifier-default/invocations",
+              "connectionType" : "INTERNET",
+              "credentials": "arn:aws:iam::{account-id}:role/{iam-role}"
+            }
+          }
+        }
+    }
+}

jobNotifier/cache/cache.go

@@ -0,0 +1,65 @@
+package cache
+
+import (
+	"jobNotifier/job"
+)
+
+type Table interface {
+	ReadItem(company string) (string, error)
+	WriteItems(companies []string)
+}
+
+type Cache struct {
+	table Table
+}
+
+func NewCache(table Table) *Cache {
+	return &Cache{table: table}
+}
+
+func (c *Cache) FilterCachedCompanies(jobs []job.Job) ([]job.Job, error) {
+	notInCache := make([]job.Job, 0)
+	errChan := make(chan error, len(jobs))
+	notFoundChan := make(chan job.Job, len(jobs))
+	foundChan := make(chan job.Job, len(jobs))
+
+	for _, newJob := range jobs {
+		go func(newJob job.Job) {
+			result, err := c.table.ReadItem(newJob.Company)
+			if result == "" {
+				// company is not in the cache
+				notFoundChan <- newJob
+			} else {
+				foundChan <- newJob
+			}
+
+			if err != nil {
+				errChan <- err
+			}
+
+		}(newJob)
+	}
+
+	// Collect results from the goroutines
+	for range jobs {
+		select {
+		case job := <-notFoundChan:
+			notInCache = append(notInCache, job)
+		case <-foundChan:
+			// do nothing
+		case err := <-errChan:
+			return nil, err
+		}
+
+	}
+
+	return notInCache, nil
+}
+
+func (c *Cache) WriteCompaniesToCache(jobs []job.Job) {
+	companies := make([]string, 0, len(jobs))
+	for _, job := range jobs {
+		companies = append(companies, job.Company)
+	}
+	c.table.WriteItems(companies)
+}

jobNotifier/cache/cache_test.go

@@ -0,0 +1,63 @@
+package cache
+
+import (
+	"jobNotifier/job"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+)
+
+type MockTable struct {
+	mock.Mock
+}
+
+func (m *MockTable) ReadItem(company string) (string, error) {
+	args := m.Called(company)
+	return args.String(0), args.Error(1)
+}
+
+func (m *MockTable) WriteItems(companies []string) {
+	m.Called(companies)
+}
+
+func TestFilterCachedCompanies(t *testing.T) {
+	mockTable := new(MockTable)
+	mockTable.On("ReadItem", "Acme Corp").Return("Acme Corp", nil)
+	mockTable.On("ReadItem", "Globex Corporation").Return("", nil)
+
+	cache := &Cache{
+		table: mockTable,
+	}
+
+	// Test the FilterCachedCompanies method
+	jobs := []job.Job{
+		{Company: "Acme Corp"},
+		{Company: "Globex Corporation"},
+	}
+	notInCache, err := cache.FilterCachedCompanies(jobs)
+
+	assert.NoError(t, err)
+	assert.Len(t, notInCache, 1)
+	assert.Equal(t, "Globex Corporation", notInCache[0].Company)
+
+	mockTable.AssertExpectations(t)
+}
+
+func TestWriteCompaniesToCache(t *testing.T) {
+	mockTable := new(MockTable)
+	mockTable.On("WriteItems", []string{"Acme Corp", "Globex Corporation"}).Return()
+
+	cache := &Cache{
+		table: mockTable,
+	}
+
+	// Test the WriteCompaniesToCache method
+	jobs := []job.Job{
+		{Company: "Acme Corp"},
+		{Company: "Globex Corporation"},
+	}
+	cache.WriteCompaniesToCache(jobs)
+
+	mockTable.AssertExpectations(t)
+}

jobNotifier/discord/discord.go

@@ -0,0 +1,72 @@
+package discord
+
+import (
+	"bytes"
+	"encoding/json"
+	"jobNotifier/job"
+	"net/http"
+)
+
+func generateMessages(jobs []job.Job) []string {
+	var messages []string
+	var message bytes.Buffer
+	message.WriteString("```")
+
+	for _, job := range jobs {
+		newLine := job.Link + ", " + job.Company + "\n"
+		// Discord has a 2000 character limit for messages
+		if message.Len()+len(newLine)+3 >= 2000 { // +3 for the ending "```"
+			message.WriteString("```")
+			messages = append(messages, message.String())
+			message.Reset()
+			message.WriteString("```")
+		}
+		message.WriteString(newLine)
+	}
+
+	if message.Len() > 0 {
+		message.WriteString("```")
+		messages = append(messages, message.String())
+	}
+
+	return messages
+}
+
+func SendJobsToDiscord(jobs []job.Job, webhookURL string) []error {
+	if len(jobs) == 0 {
+		return nil
+	}
+	messages := generateMessages(jobs)
+	errorChannel := make(chan error, len(messages))
+
+	for _, message := range messages {
+		go func(message string) {
+			payload := map[string]string{
+				"content": message,
+			}
+
+			jsonPayload, err := json.Marshal(payload)
+			if err != nil {
+				errorChannel <- err
+				return
+			}
+
+			resp, err := http.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload))
+			if err != nil {
+				errorChannel <- err
+				return
+			}
+			defer resp.Body.Close()
+			errorChannel <- nil
+		}(message)
+	}
+
+	var errors []error
+	for i := 0; i < len(messages); i++ {
+		if err := <-errorChannel; err != nil {
+			errors = append(errors, err)
+		}
+	}
+
+	return errors
+}

jobNotifier/discord/discord_test.go

@@ -0,0 +1,60 @@
+package discord
+
+import (
+	"jobNotifier/job"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGenerateMessages(t *testing.T) {
+	jobs := []job.Job{
+		{Link: "http://example.com/job1", Company: "Company1"},
+		{Link: "http://example.com/job2", Company: "Company2"},
+		{Link: "http://example.com/job3", Company: "Company3"},
+		// Add more jobs to test the 2000 character limit
+	}
+
+	messages := generateMessages(jobs)
+
+	// Check that each message is less than or equal to 2000 characters
+	for _, message := range messages {
+		assert.True(t, len(message) <= 2000, "Message length should be less than or equal to 2000 characters")
+	}
+
+	// Check that all jobs are included in the messages
+	for _, job := range jobs {
+		jobLine := job.Link + ", " + job.Company
+		found := false
+		for _, message := range messages {
+			if strings.Contains(message, jobLine) {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "All jobs should be included in the messages")
+	}
+}
+
+func TestGenerateMessages_MultipleMessages(t *testing.T) {
+	// Create a job with a link and company name that together are 200 characters long
+	newJob := job.Job{
+		Link:    strings.Repeat("a", 100), // = 100
+		Company: strings.Repeat("b", 97),  // ", " and the ending "\n" is 3 characters, so 97 + 3 = 100
+	}
+
+	// Create 11 jobs, which should result in a total length of 2200 of job text characters
+	jobs := make([]job.Job, 11)
+	for i := range jobs {
+		jobs[i] = newJob
+	}
+
+	messages := generateMessages(jobs)
+
+	// Check that multiple messages were created
+	assert.True(t, len(messages) == 2, "Multiple messages should be created when the total length of the jobs exceeds 2000 characters")
+	// The addional 6 characters are the "```" and "```" characters at the start and end of the message
+	assert.True(t, len(messages[0]) == 1806, "The first message should be 1806 characters long")
+	assert.True(t, len(messages[1]) == 406, "The second message should be 406 characters long")
+}