improve handling of errors due to maintenance

ccbrown · Dec 12, 2024 · bcc3536 · bcc3536
1 parent bde81e8
commit bcc3536
Show file tree

Hide file tree

Showing 3 changed files with 104 additions and 3 deletions.
diff --git a/server/forum_indexer.go b/server/forum_indexer.go
@@ -169,7 +169,12 @@ func (indexer *ForumIndexer) run() {
 				return
 			default:
 				if err := indexer.index(account, timezone); err != nil {
-					log.WithError(err).Error("error indexing forum account: " + account.Username)
+					if errors.Is(err, ErrForumMaintenance) {
+						log.Info("forum is under maintenance")
+						time.Sleep(30 * time.Second)
+					} else {
+						log.WithError(err).Error("error indexing forum account: " + account.Username)
+					}
 				}
 				time.Sleep(time.Second)
 			}
@@ -213,13 +218,19 @@ var postURLExpression = regexp.MustCompile("^/forum/view-post/([0-9]+)")
 var threadURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)")
 var forumURLExpression = regexp.MustCompile("^/forum/view-forum/([0-9]+)")
 
+var ErrForumMaintenance = errors.New("forum is in maintenance")
+
 func ScrapeForumPosts(doc *goquery.Document, poster ForumAccount, timezone *time.Location) ([]*ForumPost, error) {
 	posts := []*ForumPost(nil)
 
 	err := error(nil)
 
 	if doc.Find(".forumPostListTable").Length() == 0 {
-		return nil, errors.New("forum post list not found")
+		err = errors.New("forum post list not found")
+		if topBar := doc.Find(".topBar"); topBar.Length() == 1 && topBar.Text() == "Down For Maintenance" {
+			err = ErrForumMaintenance
+		}
+		return nil, err
 	}
 
 	doc.Find(".forumPostListTable > tbody > tr").EachWithBreak(func(i int, sel *goquery.Selection) bool {
@@ -292,7 +303,7 @@ func (indexer *ForumIndexer) index(poster ForumAccount, timezone *time.Location)
 	for page := 1; ; page++ {
 		posts, err := indexer.forumPosts(poster, page, timezone)
 		if err != nil {
-			logger.WithError(err).Error("error requesting forum posts")
+			return fmt.Errorf("error getting forum posts: %w", err)
 		}
 
 		done := len(posts) == 0

diff --git a/server/forum_indexer_test.go b/server/forum_indexer_test.go
@@ -53,6 +53,22 @@ func TestScrapeForumPosts(t *testing.T) {
 		require.Error(t, err)
 		assert.Equal(t, 0, len(posts))
 	})
+
+	t.Run("Maintenance", func(t *testing.T) {
+		f, err := os.Open("testdata/forum-maintenance.html")
+		require.NoError(t, err)
+		defer f.Close()
+
+		doc, err := goquery.NewDocumentFromReader(f)
+		require.NoError(t, err)
+
+		tz, err := time.LoadLocation("America/Los_Angeles")
+		require.NoError(t, err)
+
+		posts, err := ScrapeForumPosts(doc, poster, tz)
+		assert.Equal(t, err, ErrForumMaintenance)
+		assert.Equal(t, 0, len(posts))
+	})
 }
 
 func TestScrapeForumTimezone(t *testing.T) {

diff --git a/server/testdata/forum-maintenance.html b/server/testdata/forum-maintenance.html
@@ -0,0 +1,74 @@
+<!DOCTYPE html>
+<html lang="en-US">
+<head>
+    <title>Path of Exile</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
+    <link href="//poe.game.daum.net/"  hreflang="ko-KR" rel="alternate">
+    <link href="//jp.pathofexile.com/" hreflang="ja-JP" rel="alternate">
+    <link href="//es.pathofexile.com/" hreflang="es-ES" rel="alternate">
+    <link href="//de.pathofexile.com/" hreflang="de-DE" rel="alternate">
+    <link href="//fr.pathofexile.com/" hreflang="fr-FR" rel="alternate">
+    <link href="//th.pathofexile.com/" hreflang="th-TH" rel="alternate">
+    <link href="//ru.pathofexile.com/" hreflang="ru-RU" rel="alternate">
+    <link href="//br.pathofexile.com/" hreflang="pt-BR" rel="alternate">
+    <link href="//pathofexile.com/" hreflang="en" rel="alternate">
+    <link href="/favicon.ico?oriath" rel="shortcut icon">
+    <link href="/error/error_files/screen2.css?v=8" media="screen" rel="stylesheet" type="text/css">
+</head>
+<body class="en_US">
+    <div class="container">
+        <div class="header en">
+            <div class="controls">
+                <div class="language-select">
+                    <a href="//pathofexile.com/"><span class="en-lang"><img src="/error/error_files/lang/GB.png"></span></a>
+                    <a href="//br.pathofexile.com/"><span class="br-lang"><img src="/error/error_files/lang/BR.png"></span></a>
+                    <a href="//ru.pathofexile.com/"><span class="ru-lang"><img src="/error/error_files/lang/RU.png"></span></a>
+                    <a href="//th.pathofexile.com/"><span class="th-lang"><img src="/error/error_files/lang/TH.png"></span></a>
+                    <a href="//de.pathofexile.com/"><span class="de-lang"><img src="/error/error_files/lang/DE.png"></span></a>
+                    <a href="//fr.pathofexile.com/"><span class="fr-lang"><img src="/error/error_files/lang/FR.png"></span></a>
+                    <a href="//es.pathofexile.com/"><span class="es-lang"><img src="/error/error_files/lang/ES.png"></span></a>
+                    <a href="//jp.pathofexile.com/"><span class="jp-lang"><img src="/error/error_files/lang/JP.png"></span></a>
+                    <a href="//poe.game.daum.net/"><span class="kr-lang"><img src="/error/error_files/lang/KR.png"></span></a>
+                </div>
+                <a id="mainLogoLink" href="/">
+                    <span>Path of Exile</span>
+                </a>
+            </div>
+        </div>
+        <div class="content">
+            <div class="backdrop"></div>
+            <div class="wrapper">
+                <div class="layoutBox1 layoutBoxFull padded rightBackground1Theme">
+                    <h1 class="topBar first last layoutBoxTitle">Down For Maintenance</h1>
+                    <div class="layoutBoxContent">
+                        <p class="bold">
+                            pathofexile.com is currently down for maintenance. Please try again later.<br /><br />
+Thanks for your patience!
+                        </p>
+                    </div>
+                </div>
+                <div class="layoutBox1 layoutBoxFull noneTheme padded">
+                    <div class="layoutBoxContent centered">
+                        <a data-chrome="noheader nofooter transparent" data-link-color="#dac5a8" class="twitter-timeline" data-dnt="true" data-theme="dark" href="https://twitter.com/pathofexile">Tweets by pathofexile</a>
+                        <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>
+                    </div>
+                </div>
+                <div class="clear"></div>
+            </div>
+        </div>
+        <div class="footer">
+            <div class="backdrop"></div>
+            <div class="wrapper">
+                <div class="sitemap"></div>
+                <div class="bottom">
+                    <span class="copyright">&#169; 2010 - 2019 <a href="http://www.grindinggear.com">Grinding Gear Games</a></span>
+                    <span><a href="mailto:support@grindinggear.com">Contact Support</a></span><br>
+                    <a class="logo" href="http://www.grindinggear.com"><img src="/error/error_files/ggg-dark.png" alt="Grinding Gear Games"></a>
+                </div>
+                <div class="clear"></div>
+            </div>
+        </div>
+    </div>
+</body>
+</html>