-
-
Notifications
You must be signed in to change notification settings - Fork 159
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add FastMode to fetch results much faster (#102)
Utilizing scrapemate's stealth mode to try to fetch directly the results without JS rendering. This version is much faster. However the version is still in BETA. Results are not extracted in full and there is the possibility of the client being blocked
- Loading branch information
Showing
18 changed files
with
586 additions
and
62 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| package gmaps | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| olc "github.com/google/open-location-code/go" | ||
| ) | ||
|
|
||
| func ParseSearchResults(raw []byte) ([]*Entry, error) { | ||
| var data []any | ||
| if err := json.Unmarshal(raw, &data); err != nil { | ||
| return nil, fmt.Errorf("failed to unmarshal JSON: %w", err) | ||
| } | ||
|
|
||
| if len(data) == 0 { | ||
| return nil, fmt.Errorf("empty JSON data") | ||
| } | ||
|
|
||
| container, ok := data[0].([]any) | ||
| if !ok || len(container) == 0 { | ||
| return nil, fmt.Errorf("invalid business list structure") | ||
| } | ||
|
|
||
| items := getNthElementAndCast[[]any](container, 1) | ||
| if len(items) < 2 { | ||
| return nil, fmt.Errorf("empty business list") | ||
| } | ||
|
|
||
| entries := make([]*Entry, 0, len(items)-1) | ||
|
|
||
| for i := 1; i < len(items); i++ { | ||
| arr, ok := items[i].([]any) | ||
| if !ok { | ||
| continue | ||
| } | ||
|
|
||
| business := getNthElementAndCast[[]any](arr, 14) | ||
|
|
||
| var entry Entry | ||
|
|
||
| entry.ID = getNthElementAndCast[string](business, 0) | ||
| entry.Title = getNthElementAndCast[string](business, 11) | ||
| entry.Categories = toStringSlice(getNthElementAndCast[[]any](business, 13)) | ||
| entry.WebSite = getNthElementAndCast[string](business, 7, 0) | ||
|
|
||
| entry.ReviewRating = getNthElementAndCast[float64](business, 4, 7) | ||
| entry.ReviewCount = int(getNthElementAndCast[float64](business, 4, 8)) | ||
|
|
||
| fullAddress := getNthElementAndCast[[]any](business, 2) | ||
|
|
||
| entry.Address = func() string { | ||
| sb := strings.Builder{} | ||
|
|
||
| for i, part := range fullAddress { | ||
| if i > 0 { | ||
| sb.WriteString(", ") | ||
| } | ||
|
|
||
| sb.WriteString(fmt.Sprintf("%v", part)) | ||
| } | ||
|
|
||
| return sb.String() | ||
| }() | ||
|
|
||
| entry.Latitude = getNthElementAndCast[float64](business, 9, 2) | ||
| entry.Longtitude = getNthElementAndCast[float64](business, 9, 3) | ||
| entry.Phone = strings.ReplaceAll(getNthElementAndCast[string](business, 178, 0, 0), " ", "") | ||
| entry.OpenHours = getHours(business) | ||
| entry.Status = getNthElementAndCast[string](business, 34, 4, 4) | ||
| entry.Timezone = getNthElementAndCast[string](business, 30) | ||
| entry.DataID = getNthElementAndCast[string](business, 10) | ||
|
|
||
| entry.PlusCode = olc.Encode(entry.Latitude, entry.Longtitude, 10) | ||
|
|
||
| entries = append(entries, &entry) | ||
| } | ||
|
|
||
| return entries, nil | ||
| } | ||
|
|
||
| func toStringSlice(arr []any) []string { | ||
| ans := make([]string, 0, len(arr)) | ||
| for _, v := range arr { | ||
| ans = append(ans, fmt.Sprintf("%v", v)) | ||
| } | ||
|
|
||
| return ans | ||
| } |
Oops, something went wrong.