From 4fb833970731de638eef9a89f32b56dbc3317cdf Mon Sep 17 00:00:00 2001 From: Austin Davis Date: Thu, 18 Apr 2024 20:47:22 -0600 Subject: [PATCH 1/2] start of the jobNotifier lambda --- go.work | 1 + go.work.sum | 2 + jobNotifier/cache/cache.go | 65 +++++++++++++ jobNotifier/cache/cache_test.go | 63 ++++++++++++ jobNotifier/discord/discord.go | 72 ++++++++++++++ jobNotifier/discord/discord_test.go | 60 ++++++++++++ jobNotifier/dynamo/dynamo.go | 98 +++++++++++++++++++ jobNotifier/dynamo/dynamo_test.go | 52 ++++++++++ jobNotifier/go.mod | 17 ++++ jobNotifier/go.sum | 24 +++++ jobNotifier/job/job.go | 8 ++ jobNotifier/main.go | 142 ++++++++++++++++++++++++++++ 12 files changed, 604 insertions(+) create mode 100644 jobNotifier/cache/cache.go create mode 100644 jobNotifier/cache/cache_test.go create mode 100644 jobNotifier/discord/discord.go create mode 100644 jobNotifier/discord/discord_test.go create mode 100644 jobNotifier/dynamo/dynamo.go create mode 100644 jobNotifier/dynamo/dynamo_test.go create mode 100644 jobNotifier/go.mod create mode 100644 jobNotifier/go.sum create mode 100644 jobNotifier/job/job.go create mode 100644 jobNotifier/main.go diff --git a/go.work b/go.work index e8f7ce2..1790922 100644 --- a/go.work +++ b/go.work @@ -3,4 +3,5 @@ go 1.21.6 use ( ./proxy ./scraper + ./jobNotifier ) diff --git a/go.work.sum b/go.work.sum index 12c5361..e153131 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,2 +1,4 @@ +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= diff --git a/jobNotifier/cache/cache.go b/jobNotifier/cache/cache.go new file mode 100644 index 0000000..6deaa7f --- /dev/null +++ b/jobNotifier/cache/cache.go @@ -0,0 +1,65 @@ +package cache + +import ( + "jobNotifier/job" +) + +type Table interface { + ReadItem(company string) (string, error) + WriteItems(companies []string) +} + +type Cache struct { + table Table +} + +func NewCache(table Table) *Cache { + return &Cache{table: table} +} + +func (c *Cache) FilterCachedCompanies(jobs []job.Job) ([]job.Job, error) { + notInCache := make([]job.Job, 0) + errChan := make(chan error, len(jobs)) + notFoundChan := make(chan job.Job, len(jobs)) + foundChan := make(chan job.Job, len(jobs)) + + for _, newJob := range jobs { + go func(newJob job.Job) { + result, err := c.table.ReadItem(newJob.Company) + if result == "" { + // company is not in the cache + notFoundChan <- newJob + } else { + foundChan <- newJob + } + + if err != nil { + errChan <- err + } + + }(newJob) + } + + // Collect results from the goroutines + for range jobs { + select { + case job := <-notFoundChan: + notInCache = append(notInCache, job) + case <-foundChan: + // do nothing + case err := <-errChan: + return nil, err + } + + } + + return notInCache, nil +} + +func (c *Cache) WriteCompaniesToCache(jobs []job.Job) { + companies := make([]string, 0, len(jobs)) + for _, job := range jobs { + companies = append(companies, job.Company) + } + c.table.WriteItems(companies) +} diff --git a/jobNotifier/cache/cache_test.go b/jobNotifier/cache/cache_test.go new file mode 100644 index 0000000..8d6407b --- /dev/null +++ b/jobNotifier/cache/cache_test.go @@ -0,0 +1,63 @@ +package cache + +import ( + "jobNotifier/job" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type MockTable struct { + mock.Mock +} + +func (m *MockTable) ReadItem(company string) (string, error) { + args := m.Called(company) + return args.String(0), args.Error(1) +} + +func (m *MockTable) WriteItems(companies []string) { + m.Called(companies) +} + +func TestFilterCachedCompanies(t *testing.T) { + mockTable := new(MockTable) + mockTable.On("ReadItem", "Acme Corp").Return("Acme Corp", nil) + mockTable.On("ReadItem", "Globex Corporation").Return("", nil) + + cache := &Cache{ + table: mockTable, + } + + // Test the FilterCachedCompanies method + jobs := []job.Job{ + {Company: "Acme Corp"}, + {Company: "Globex Corporation"}, + } + notInCache, err := cache.FilterCachedCompanies(jobs) + + assert.NoError(t, err) + assert.Len(t, notInCache, 1) + assert.Equal(t, "Globex Corporation", notInCache[0].Company) + + mockTable.AssertExpectations(t) +} + +func TestWriteCompaniesToCache(t *testing.T) { + mockTable := new(MockTable) + mockTable.On("WriteItems", []string{"Acme Corp", "Globex Corporation"}).Return() + + cache := &Cache{ + table: mockTable, + } + + // Test the WriteCompaniesToCache method + jobs := []job.Job{ + {Company: "Acme Corp"}, + {Company: "Globex Corporation"}, + } + cache.WriteCompaniesToCache(jobs) + + mockTable.AssertExpectations(t) +} diff --git a/jobNotifier/discord/discord.go b/jobNotifier/discord/discord.go new file mode 100644 index 0000000..ccc7277 --- /dev/null +++ b/jobNotifier/discord/discord.go @@ -0,0 +1,72 @@ +package discord + +import ( + "bytes" + "encoding/json" + "jobNotifier/job" + "net/http" +) + +func generateMessages(jobs []job.Job) []string { + var messages []string + var message bytes.Buffer + message.WriteString("```") + + for _, job := range jobs { + newLine := job.Link + ", " + job.Company + "\n" + // Discord has a 2000 character limit for messages + if message.Len()+len(newLine)+3 >= 2000 { // +3 for the ending "```" + message.WriteString("```") + messages = append(messages, message.String()) + message.Reset() + message.WriteString("```") + } + message.WriteString(newLine) + } + + if message.Len() > 0 { + message.WriteString("```") + messages = append(messages, message.String()) + } + + return messages +} + +func SendJobsToDiscord(jobs []job.Job, webhookURL string) []error { + if len(jobs) == 0 { + return nil + } + messages := generateMessages(jobs) + errorChannel := make(chan error, len(messages)) + + for _, message := range messages { + go func(message string) { + payload := map[string]string{ + "content": message, + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + errorChannel <- err + return + } + + resp, err := http.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + errorChannel <- err + return + } + defer resp.Body.Close() + errorChannel <- nil + }(message) + } + + var errors []error + for i := 0; i < len(messages); i++ { + if err := <-errorChannel; err != nil { + errors = append(errors, err) + } + } + + return errors +} diff --git a/jobNotifier/discord/discord_test.go b/jobNotifier/discord/discord_test.go new file mode 100644 index 0000000..69c1667 --- /dev/null +++ b/jobNotifier/discord/discord_test.go @@ -0,0 +1,60 @@ +package discord + +import ( + "jobNotifier/job" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGenerateMessages(t *testing.T) { + jobs := []job.Job{ + {Link: "http://example.com/job1", Company: "Company1"}, + {Link: "http://example.com/job2", Company: "Company2"}, + {Link: "http://example.com/job3", Company: "Company3"}, + // Add more jobs to test the 2000 character limit + } + + messages := generateMessages(jobs) + + // Check that each message is less than or equal to 2000 characters + for _, message := range messages { + assert.True(t, len(message) <= 2000, "Message length should be less than or equal to 2000 characters") + } + + // Check that all jobs are included in the messages + for _, job := range jobs { + jobLine := job.Link + ", " + job.Company + found := false + for _, message := range messages { + if strings.Contains(message, jobLine) { + found = true + break + } + } + assert.True(t, found, "All jobs should be included in the messages") + } +} + +func TestGenerateMessages_MultipleMessages(t *testing.T) { + // Create a job with a link and company name that together are 200 characters long + newJob := job.Job{ + Link: strings.Repeat("a", 100), // = 100 + Company: strings.Repeat("b", 97), // ", " and the ending "\n" is 3 characters, so 97 + 3 = 100 + } + + // Create 11 jobs, which should result in a total length of 2200 of job text characters + jobs := make([]job.Job, 11) + for i := range jobs { + jobs[i] = newJob + } + + messages := generateMessages(jobs) + + // Check that multiple messages were created + assert.True(t, len(messages) == 2, "Multiple messages should be created when the total length of the jobs exceeds 2000 characters") + // The addional 6 characters are the "```" and "```" characters at the start and end of the message + assert.True(t, len(messages[0]) == 1806, "The first message should be 1806 characters long") + assert.True(t, len(messages[1]) == 406, "The second message should be 406 characters long") +} diff --git a/jobNotifier/dynamo/dynamo.go b/jobNotifier/dynamo/dynamo.go new file mode 100644 index 0000000..39b00d1 --- /dev/null +++ b/jobNotifier/dynamo/dynamo.go @@ -0,0 +1,98 @@ +package dynamo + +import ( + "log" + "strconv" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/dynamodb" +) + +type DynamoDBAPI interface { + UpdateItem(input *dynamodb.UpdateItemInput) (*dynamodb.UpdateItemOutput, error) + GetItem(input *dynamodb.GetItemInput) (*dynamodb.GetItemOutput, error) +} + +type Table struct { + Name string + svc DynamoDBAPI +} + +func NewTable(name string, region string) (*Table, error) { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(region), // replace with your region + }) + if err != nil { + return nil, err + } + + svc := dynamodb.New(sess) + + return &Table{Name: name, svc: svc}, nil +} + +func (t *Table) ReadItem(company string) (string, error) { + input := &dynamodb.GetItemInput{ + TableName: aws.String(t.Name), + Key: map[string]*dynamodb.AttributeValue{ + "company": { + S: aws.String(strings.ToLower(company)), + }, + }, + } + + result, err := t.svc.GetItem(input) + if err != nil { + return "", err + } + + if result.Item == nil { + return "", nil + } + + return *result.Item["company"].S, nil +} + +func (t *Table) WriteItems(companies []string) { + // Set the ttl time to 30 days from now + expirationTime := time.Now().AddDate(0, 1, 0).Unix() + + // Create a wait group + var wg sync.WaitGroup + + // Write each company to the table in a separate goroutine + for _, company := range companies { + wg.Add(1) + go func(company string) { + defer wg.Done() + + input := &dynamodb.UpdateItemInput{ + ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ + ":expirationTime": { + N: aws.String(strconv.FormatInt(expirationTime, 10)), + }, + }, + TableName: aws.String(t.Name), + Key: map[string]*dynamodb.AttributeValue{ + "company": { + S: aws.String(strings.ToLower(company)), + }, + }, + ReturnValues: aws.String("UPDATED_NEW"), + UpdateExpression: aws.String("set ExpirationTime = :expirationTime"), + } + + _, err := t.svc.UpdateItem(input) + if err != nil { + log.Println("Error writing company to cache", err) + } + }(company) + } + + // Wait for all goroutines to finish + wg.Wait() +} diff --git a/jobNotifier/dynamo/dynamo_test.go b/jobNotifier/dynamo/dynamo_test.go new file mode 100644 index 0000000..ed8be07 --- /dev/null +++ b/jobNotifier/dynamo/dynamo_test.go @@ -0,0 +1,52 @@ +package dynamo + +import ( + "testing" + + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type MockDynamoDB struct { + mock.Mock +} + +func (m *MockDynamoDB) UpdateItem(input *dynamodb.UpdateItemInput) (*dynamodb.UpdateItemOutput, error) { + args := m.Called(input) + return args.Get(0).(*dynamodb.UpdateItemOutput), args.Error(1) +} + +func (m *MockDynamoDB) GetItem(input *dynamodb.GetItemInput) (*dynamodb.GetItemOutput, error) { + args := m.Called(input) + return args.Get(0).(*dynamodb.GetItemOutput), args.Error(1) +} + +func TestNewTable(t *testing.T) { + table, err := NewTable("test", "us-west-2") + assert.NoError(t, err) + assert.NotNil(t, table) +} + +func TestReadItem(t *testing.T) { + mockSvc := new(MockDynamoDB) + table := &Table{Name: "test", svc: mockSvc} + + mockSvc.On("GetItem", mock.Anything).Return(&dynamodb.GetItemOutput{}, nil) + + _, err := table.ReadItem("Acme Corp") + assert.NoError(t, err) + + mockSvc.AssertExpectations(t) +} + +func TestWriteItems(t *testing.T) { + mockSvc := new(MockDynamoDB) + table := &Table{Name: "test", svc: mockSvc} + + mockSvc.On("UpdateItem", mock.Anything).Return(&dynamodb.UpdateItemOutput{}, nil) + + table.WriteItems([]string{"Acme Corp", "Globex Corporation"}) + + mockSvc.AssertExpectations(t) +} diff --git a/jobNotifier/go.mod b/jobNotifier/go.mod new file mode 100644 index 0000000..c11bc6d --- /dev/null +++ b/jobNotifier/go.mod @@ -0,0 +1,17 @@ +module jobNotifier + +go 1.21.6 + +require ( + github.com/aws/aws-lambda-go v1.47.0 + github.com/aws/aws-sdk-go v1.51.24 + github.com/stretchr/testify v1.9.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/jobNotifier/go.sum b/jobNotifier/go.sum new file mode 100644 index 0000000..b25d306 --- /dev/null +++ b/jobNotifier/go.sum @@ -0,0 +1,24 @@ +github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= +github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= +github.com/aws/aws-sdk-go v1.51.24 h1:nwL5MaommPkwb7Ixk24eWkdx5HY4of1gD10kFFVAl6A= +github.com/aws/aws-sdk-go v1.51.24/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/jobNotifier/job/job.go b/jobNotifier/job/job.go new file mode 100644 index 0000000..e3f87bd --- /dev/null +++ b/jobNotifier/job/job.go @@ -0,0 +1,8 @@ +package job + +type Job struct { + Title string `json:"title"` + Company string `json:"company"` + Keyword string `json:"keyword"` + Link string `json:"link"` +} diff --git a/jobNotifier/main.go b/jobNotifier/main.go new file mode 100644 index 0000000..d8ca726 --- /dev/null +++ b/jobNotifier/main.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "jobNotifier/cache" + "jobNotifier/discord" + "jobNotifier/dynamo" + "jobNotifier/job" + "log" + "os" + + "github.com/aws/aws-lambda-go/events" + "github.com/aws/aws-lambda-go/lambda" +) + +type RequestBody struct { + Jobs []job.Job `json:"jobs"` +} + +type Results struct { + Total int `json:"total"` + Uncached int `json:"uncached"` + Duplicates int `json:"duplicates"` +} + +var ( + scraperWebhook string + dynamoTable string +) + +func init() { + scraperWebhook = os.Getenv("SCRAPER_WEBHOOK") + if scraperWebhook == "" { + log.Fatal("Environment variable SCRAPER_WEBHOOK must be set") + } + + dynamoTable = os.Getenv("DYNAMO_TABLE") + if dynamoTable == "" { + log.Fatal("Environment variable DYNAMO_TABLE must be set") + } + +} + +func main() { + if os.Getenv("_LAMBDA_SERVER_PORT") == "" && os.Getenv("AWS_LAMBDA_RUNTIME_API") == "" { + offlineHandler() + } else { + lambda.Start(handler) + } +} + +func notifyNewJobs(jobs []job.Job) (Results, error) { + results := Results{} + results.Total = len(jobs) + table, err := dynamo.NewTable(dynamoTable, "us-east-1") // replace with your table name + if err != nil { + log.Fatal(err) + } + cache := cache.NewCache(table) + unCachedJobs, err := cache.FilterCachedCompanies(jobs) + results.Uncached = len(unCachedJobs) + results.Duplicates = results.Total - results.Uncached + if err != nil { + return results, err + } + errs := discord.SendJobsToDiscord(unCachedJobs, scraperWebhook) + if len(errs) == 0 { + cache.WriteCompaniesToCache(unCachedJobs) + } else { + return results, fmt.Errorf("error sending to discord %v", errs) + } + return results, nil +} + +func handler(ctx context.Context, request events.APIGatewayV2HTTPRequest) (events.APIGatewayProxyResponse, error) { + var requestBody RequestBody + err := json.Unmarshal([]byte(request.Body), &requestBody) + if err != nil { + return events.APIGatewayProxyResponse{ + StatusCode: 400, + Body: "Invalid request body", + }, nil + } + + results, err := notifyNewJobs(requestBody.Jobs) + if err != nil { + return events.APIGatewayProxyResponse{ + StatusCode: 500, + Body: "Failed to notify new jobs", + }, nil + } + + resultsBytes, err := json.Marshal(results) + if err != nil { + return events.APIGatewayProxyResponse{ + StatusCode: 500, + Body: "Failed to convert results to JSON", + }, nil + } + + return events.APIGatewayProxyResponse{ + StatusCode: 200, + Body: string(resultsBytes), + }, nil +} + +func offlineHandler() { + mockJobs := []job.Job{ + { + Title: "Software Engineer", + Company: "test1", + Keyword: "Go", + Link: "https://testlink1.com", + }, + { + Title: "Data Analyst", + Company: "test2", + Keyword: "Go", + Link: "https://testlink1.com", + }, + { + Title: "Financial Advisor", + Company: "test3", + Keyword: "Go", + Link: "https://testlink3.com", + }, + { + Title: "Educational Consultant", + Company: "test4", + Keyword: "Go", + Link: "https://testlink4.com", + }, + } + results, err := notifyNewJobs(mockJobs) + if err != nil { + fmt.Println(err) + } else { + fmt.Println("Total", results.Total, "Uncached", results.Uncached, "Duplicates", results.Duplicates) + } +} From 30a9198b31018702a35e8dc490121e1f9f2b53f4 Mon Sep 17 00:00:00 2001 From: Austin Davis Date: Sat, 20 Apr 2024 19:48:13 -0600 Subject: [PATCH 2/2] terraform/action for job api/lambda --- .github/workflows/pr.yml | 8 ++++++++ Makefile | 5 ++++- job-openapi.json | 26 ++++++++++++++++++++++++++ openapi.json => proxy-openapi.json | 0 terraform/main.tf | 24 ++++++++++++++++++++++-- 5 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 job-openapi.json rename openapi.json => proxy-openapi.json (100%) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 029b45b..4461d41 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -41,6 +41,14 @@ jobs: run: go build -v ./... working-directory: proxy + - name: Build jobNotifier + run: go build -v ./... + working-directory: jobNotifier + + - name: Test jobNotifier + run: go test -v ./... + working-directory: jobNotifier + - name: Setup Node.js uses: actions/setup-node@v4 with: diff --git a/Makefile b/Makefile index 5bc1edd..ab1c631 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # AWS go lambdas running on provided.al2 runtime have to be called bootstrap # https://docs.aws.amazon.com/lambda/latest/dg/golang-handler.html#golang-handler-naming -packageLambdas: packageScraper packageProxy +packageLambdas: packageScraper packageProxy packageJobNotifier packageScraper: cd scraper && GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -tags lambda.norpc -o bootstrap main.go && zip bootstrap.zip bootstrap && rm bootstrap && ls @@ -9,4 +9,7 @@ packageScraper: packageProxy: cd proxy && GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -tags lambda.norpc -o bootstrap main.go && zip bootstrap.zip bootstrap && rm bootstrap && ls +packageJobNotifier: + cd jobNotifier && GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -tags lambda.norpc -o bootstrap main.go && zip bootstrap.zip bootstrap && rm bootstrap && ls + diff --git a/job-openapi.json b/job-openapi.json new file mode 100644 index 0000000..5a74762 --- /dev/null +++ b/job-openapi.json @@ -0,0 +1,26 @@ +{ + "openapi": "3.0.1", + "info": { + "title": "Job API", + "version": "1.0" + }, + "paths" : { + "/job" : { + "post" : { + "responses" : { + "default" : { + "description" : "Default response for POST /job" + } + }, + "x-amazon-apigateway-integration" : { + "payloadFormatVersion" : "2.0", + "type" : "aws_proxy", + "httpMethod" : "POST", + "uri" : "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:{account-id}:function:jobNotifier-default/invocations", + "connectionType" : "INTERNET", + "credentials": "arn:aws:iam::{account-id}:role/{iam-role}" + } + } + } + } +} \ No newline at end of file diff --git a/openapi.json b/proxy-openapi.json similarity index 100% rename from openapi.json rename to proxy-openapi.json diff --git a/terraform/main.tf b/terraform/main.tf index 20ae1fe..447034d 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -59,6 +59,19 @@ module "headless_lambda" { env_vars = {} } +module "job_notifier_lambda" { + source = "./lambda" + zip_location = "../jobNotifier/bootstrap.zip" + name = "jobNotifier-${terraform.workspace}" + handler = "bootstrap" + run_time = "provided.al2" + timeout = 300 + env_vars = { + "DYNAMO_TABLE" = "${aws_dynamodb_table.job_scraper_company_cache.name}" + "SCRAPER_WEBHOOK" = "${var.SCRAPER_WEBHOOK}" + } +} + # --------------------------------------------------------------------------------------------------------------------- # Cloudwatch that will trigger the scraper lambda # --------------------------------------------------------------------------------------------------------------------- @@ -79,10 +92,17 @@ module "scraper_lambda_trigger" { module "proxy_gateway" { source = "./api-gateway" api_name = "proxy-${terraform.workspace}" - openapi = "../openapi.json" + openapi = "../proxy-openapi.json" lambda_arns = [ module.headless_lambda.arn, module.proxy_lambda.arn] } +module "job_gateway" { + source = "./api-gateway" + api_name = "job-${terraform.workspace}" + openapi = "../job-openapi.json" + lambda_arns = [ module.job_notifier_lambda.arn] +} + # --------------------------------------------------------------------------------------------------------------------- # DynamoDb # --------------------------------------------------------------------------------------------------------------------- @@ -109,5 +129,5 @@ module "dynamodb_lambda_iam" { source = "./dynamodb-lambda-iam" dynamodb_name = aws_dynamodb_table.job_scraper_company_cache.name dynamodb_arn = aws_dynamodb_table.job_scraper_company_cache.arn - lambda_roles = [module.scraper_lambda.role_name] + lambda_roles = [module.scraper_lambda.role_name, module.job_notifier_lambda.role_name] } \ No newline at end of file