diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 4723e15..ab422dc 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -169,7 +169,12 @@ func (indexer *ForumIndexer) run() { return default: if err := indexer.index(account, timezone); err != nil { - log.WithError(err).Error("error indexing forum account: " + account.Username) + if errors.Is(err, ErrForumMaintenance) { + log.Info("forum is under maintenance") + time.Sleep(30 * time.Second) + } else { + log.WithError(err).Error("error indexing forum account: " + account.Username) + } } time.Sleep(time.Second) } @@ -213,13 +218,19 @@ var postURLExpression = regexp.MustCompile("^/forum/view-post/([0-9]+)") var threadURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)") var forumURLExpression = regexp.MustCompile("^/forum/view-forum/([0-9]+)") +var ErrForumMaintenance = errors.New("forum is in maintenance") + func ScrapeForumPosts(doc *goquery.Document, poster ForumAccount, timezone *time.Location) ([]*ForumPost, error) { posts := []*ForumPost(nil) err := error(nil) if doc.Find(".forumPostListTable").Length() == 0 { - return nil, errors.New("forum post list not found") + err = errors.New("forum post list not found") + if topBar := doc.Find(".topBar"); topBar.Length() == 1 && topBar.Text() == "Down For Maintenance" { + err = ErrForumMaintenance + } + return nil, err } doc.Find(".forumPostListTable > tbody > tr").EachWithBreak(func(i int, sel *goquery.Selection) bool { @@ -292,7 +303,7 @@ func (indexer *ForumIndexer) index(poster ForumAccount, timezone *time.Location) for page := 1; ; page++ { posts, err := indexer.forumPosts(poster, page, timezone) if err != nil { - logger.WithError(err).Error("error requesting forum posts") + return fmt.Errorf("error getting forum posts: %w", err) } done := len(posts) == 0 diff --git a/server/forum_indexer_test.go b/server/forum_indexer_test.go index 4d48f9e..4e6b756 100644 --- a/server/forum_indexer_test.go +++ b/server/forum_indexer_test.go @@ -53,6 +53,22 @@ func TestScrapeForumPosts(t *testing.T) { require.Error(t, err) assert.Equal(t, 0, len(posts)) }) + + t.Run("Maintenance", func(t *testing.T) { + f, err := os.Open("testdata/forum-maintenance.html") + require.NoError(t, err) + defer f.Close() + + doc, err := goquery.NewDocumentFromReader(f) + require.NoError(t, err) + + tz, err := time.LoadLocation("America/Los_Angeles") + require.NoError(t, err) + + posts, err := ScrapeForumPosts(doc, poster, tz) + assert.Equal(t, err, ErrForumMaintenance) + assert.Equal(t, 0, len(posts)) + }) } func TestScrapeForumTimezone(t *testing.T) { diff --git a/server/testdata/forum-maintenance.html b/server/testdata/forum-maintenance.html new file mode 100644 index 0000000..6e1f145 --- /dev/null +++ b/server/testdata/forum-maintenance.html @@ -0,0 +1,74 @@ + + + + Path of Exile + + + + + + + + + + + + + + + +
+
+
+
+ + + + + + + + + +
+ + Path of Exile + +
+
+
+
+
+
+

Down For Maintenance

+
+

+ pathofexile.com is currently down for maintenance. Please try again later.

+Thanks for your patience! +

+
+
+
+
+ + +
+
+
+
+
+
+
+
+
+
+ © 2010 - 2019 Grinding Gear Games + Contact Support
+ +
+
+
+
+
+ +