Merge pull request #13 from austin1237/duplicate-links

All sites now deduplicate links before grabbing individual posts.
austin1237 · Feb 27, 2024 · 4d04af2 · 4d04af2
2 parents bf674d7 + 38e27a3
commit 4d04af2
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/scraper/interest/interest.go b/scraper/interest/interest.go
@@ -24,15 +24,16 @@ func CheckIfInterested(description string) bool {
 
 type JobInfoGetter func(string, string) (string, error)
 
-func FilterInterest(proxyUrl string, Jobs []job.Job, jobInfoGetter JobInfoGetter) []job.Job {
+func FilterInterest(proxyUrl string, possibleJobs []job.Job, jobInfoGetter JobInfoGetter) []job.Job {
 	interestingJobs := []job.Job{}
+	possibleJobs = job.DeduplicatedLinks(possibleJobs)
 	var wg sync.WaitGroup
 
 	// Number of concurrent goroutines
 	maxGoroutines := 10
 	var goroutineCount int
 
-	for _, possibleJob := range Jobs {
+	for _, possibleJob := range possibleJobs {
 		wg.Add(1)
 		goroutineCount++
 

diff --git a/scraper/sitec/sitec.go b/scraper/sitec/sitec.go
@@ -65,7 +65,6 @@ func ScanNewJobs(sitecBaseUrl string, proxyUrl string) []job.Job {
 		jobs = append(jobs, <-jobChannel...)
 	}
 
-	jobs = job.DeduplicatedLinks(jobs)
 	log.Println(sitecBaseUrl+" total jobs found", len(jobs))
 	interestingJobs := interest.FilterInterest(proxyUrl, jobs, getSiteCJobInfo)
 	log.Println(sitecBaseUrl+" interesting jobs", len(interestingJobs))