Bump jinja2 from 2.11.2 to 3.1.2

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10

configuration.py

@@ -0,0 +1,65 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import json
+import re
+from urllib import urlopen
+
+from functions import EVENTS
+
+
+def reData(txt, year):
+    """
+    Parser para linha da configuração
+    """
+    events = '|'.join(EVENTS)
+    regex = ur'''
+        \s*wl\["(?P<event>%s)"\]\[(?P<year>20\d\d)]\ ?=\ ?\{|
+        \s*\["(?P<country>[-a-z]+)"\]\ =\ \{\["start"\]\ =\ (?P<start>%s\d{10}),\ \["end"\]\ =\ (?P<end>%s\d\d{10})\}
+        ''' % (events, year, str(year)[:3])
+    m = re.search(regex, txt, re.X)
+    return m and m.groupdict()
+
+
+def re_prefix(txt):
+    return re.search(r'\s*\["(?P<prefix>[\w-]+)"\] = "(?P<name>[\w\-\' ]+)"|(?P<close>\})', txt, re.UNICODE)
+
+
+def get_config_from_commons(page):
+    api = urlopen('https://commons.wikimedia.org/w/api.php?action=query&format=json&prop=revisions&titles=%s&rvprop=content' % page)
+    text = json.loads(api.read())['query']['pages'].values()[0]['revisions'][0]['*']
+    return unicode(text)
+
+
+def parse_config(text):
+    data, event, prefixes = {}, None, {}
+    lines = iter(text.split(u'\n'))
+    for line in lines:
+        m = re_prefix(line)
+        if prefixes and m and m.group('close'):
+            break
+        elif m and m.group('prefix'):
+            prefixes[m.group('prefix')] = m.group('name')
+
+    for line in lines:
+        g = reData(line, event[-4:] if event else ur'20\d\d')
+        if not g:
+            continue
+        if g['event']:
+            event = g['event'] + g['year']
+            data[event] = {}
+        elif g['country'] and event:
+            if g['country'] not in prefixes:
+                # updateLog.append(u'Unknown prefix: ' + g['country'])
+                continue
+            data[event][prefixes[g['country']]] = {'start': int(g['start']), 'end': int(g['end'])}
+
+    return {name: config for name, config in data.items() if config}
+
+
+def getConfig(page):
+    """
+    Lê a configuração da página de configuração no Commons
+    """
+    text = get_config_from_commons(page)
+    return parse_config(text)

database.py

@@ -3,73 +3,14 @@
 
 import io
 import json
-import re
 import time
-from urllib import urlopen
 
+from configuration import getConfig
 from commons_database import DB
-from functions import EVENTS, get_wikiloves_category_name
+from functions import get_wikiloves_category_name
 
 updateLog = []
 
-
-def reData(txt, year):
-    """
-    Parser para linha da configuração
-    """
-    events = '|'.join(EVENTS)
-    regex = ur'''
-        \s*wl\["(?P<event>%s)"\]\[(?P<year>20\d\d)]\ ?=\ ?\{|
-        \s*\["(?P<country>[-a-z]+)"\]\ =\ \{\["start"\]\ =\ (?P<start>%s\d{10}),\ \["end"\]\ =\ (?P<end>%s\d\d{10})\}
-        ''' % (events, year, str(year)[:3])
-    m = re.search(regex, txt, re.X)
-    return m and m.groupdict()
-
-
-def re_prefix(txt):
-    return re.search(r'\s*\["(?P<prefix>[\w-]+)"\] = "(?P<name>[\w\-\' ]+)"|(?P<close>\})', txt, re.UNICODE)
-
-
-def get_config_from_commons(page):
-    api = urlopen('https://commons.wikimedia.org/w/api.php?action=query&format=json&prop=revisions&titles=%s&rvprop=content' % page)
-    text = json.loads(api.read())['query']['pages'].values()[0]['revisions'][0]['*']
-    return unicode(text)
-
-
-def parse_config(text):
-    data, event, prefixes = {}, None, {}
-    lines = iter(text.split(u'\n'))
-    for line in lines:
-        m = re_prefix(line)
-        if prefixes and m and m.group('close'):
-            break
-        elif m and m.group('prefix'):
-            prefixes[m.group('prefix')] = m.group('name')
-
-    for line in lines:
-        g = reData(line, event[-4:] if event else ur'20\d\d')
-        if not g:
-            continue
-        if g['event']:
-            event = g['event'] + g['year']
-            data[event] = {}
-        elif g['country'] and event:
-            if g['country'] not in prefixes:
-                updateLog.append(u'Unknown prefix: ' + g['country'])
-                continue
-            data[event][prefixes[g['country']]] = {'start': int(g['start']), 'end': int(g['end'])}
-
-    return {name: config for name, config in data.items() if config}
-
-
-def getConfig(page):
-    """
-    Lê a configuração da página de configuração no Commons
-    """
-    text = get_config_from_commons(page)
-    return parse_config(text)
-
-
 dbquery = u'''SELECT
  img_timestamp,
  img_name IN (SELECT DISTINCT gil_to FROM globalimagelinks) AS image_in_use,

docker-compose.yml

@@ -17,6 +17,7 @@ services:
       - DB_USERNAME=$TOOLFORGE_DB_USERNAME
       - DB_PASSWORD=$TOOLFORGE_DB_PASSWORD
       - DB_HOST=db_commons
+      - SOURCE_PATH=/code/
   db_commons:
     image: jeanfred/wiki-replicas-proxy
     volumes:

functions.py

@@ -135,6 +135,7 @@ def get_event_category_template():
     ("Monuments", "2019", "Austria"): 'Media_from_WikiDaheim_2019_in_Austria/Cultural_heritage_monuments',
     ("Monuments", "2018", "Austria"): 'Media_from_WikiDaheim_2018_in_Austria/Cultural_heritage_monuments',
     ("Monuments", "2017", "Austria"): 'Media_from_WikiDaheim_2017_in_Austria/Cultural_heritage_monuments',
+    ("Monuments", "2021", "Armenia"): 'Images_from_Wiki_Loves_Monuments_2021_in_Armenia',
     ("Monuments", "2020", "Armenia"): 'Images_from_Wiki_Loves_Monuments_2020_in_Armenia',
     ("Monuments", "2019", "Armenia"): 'Images_from_Wiki_Loves_Monuments_2019_in_Armenia',
     ("Monuments", "2013", "Armenia"): 'Images_from_Wiki_Loves_Monuments_2013_in_Armenia',

requirements.txt

@@ -2,7 +2,7 @@
 click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 flask==1.1.2
 itsdangerous==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-jinja2==2.11.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+jinja2==3.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 markupsafe==1.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 pymysql==0.10.0
 werkzeug==1.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

tests/test_configuration.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8  -*-
+"""Unit tests for configuration.py."""
+
+import unittest
+
+import configuration
+
+
+class TestReData(unittest.TestCase):
+
+    def test_reData_event_line(self):
+        input_data = 'wl["monuments"][2010] = {'
+        result = configuration.reData(input_data, 2014)
+        expected = {
+            u'country': None,
+            u'year': '2010',
+            u'end': None,
+            u'event': 'monuments',
+            u'start': None
+        }
+        self.assertEqual(result, expected)
+
+    def test_reData_event_line_public_art(self):
+        input_data = 'wl["public_art"][2012] = {'
+        result = configuration.reData(input_data, 2014)
+        expected = {
+            u'country': None,
+            u'year': '2012',
+            u'end': None,
+            u'event': 'public_art',
+            u'start': None
+        }
+        self.assertEqual(result, expected)
+
+    def test_reData_country_line(self):
+        input_data = '''
+    ["az"] = {["start"] = 20170430200000, ["end"] = 20170531195959},
+    '''
+        result = configuration.reData(input_data, 2017)
+        expected = {
+            u'country': 'az',
+            u'year': None,
+            u'end': '20170531195959',
+            u'event': None,
+            u'start': '20170430200000'
+        }
+        self.assertEqual(result, expected)
+
+
+class TestRePrefix(unittest.TestCase):
+
+    def test_re_prefix_match_ascii_line(self):
+        self.assertIsNotNone(configuration.re_prefix(u'    ["az"] = "Azerbaijan",'))
+
+    def test_re_prefix_match_ascii_line_with_space(self):
+        self.assertIsNotNone(configuration.re_prefix(u'    ["gq"] = "Equatorial Guinea",'))
+
+    def test_re_prefix_match_ascii_line_with_dash(self):
+        self.assertIsNotNone(configuration.re_prefix(u'    ["gw"] = "Guinea-Bissau",'))
+
+    def test_re_prefix_match_ascii_line_with_accents(self):
+        self.assertIsNotNone(configuration.re_prefix(u'    ["re"] = "Réunion",'))
+
+    def test_re_prefix_match_ascii_line_with_apostrophe(self):
+        self.assertIsNotNone(configuration.re_prefix(u'    ["ci"] = "Côte d\'Ivoire",'))
+
+
+class TestParseConfig(unittest.TestCase):
+
+    def test_parse_config_empty(self):
+        config = ''
+        result = configuration.parse_config(config)
+        expected = {}
+        self.assertEquals(result, expected)
+
+    def test_parse_config(self):
+        config = '''
+wl["prefixes"] = {
+    ["az"] = "Azerbaijan",
+    ["gw"] = "Guinea-Bissau"
+}
+
+wl["monuments"][2017] = {
+    ["az"] = {["start"] = 20170430200000, ["end"] = 20170531195959},
+    ["gw"] = {["start"] = 20170430200000, ["end"] = 20170531195959},
+}
+
+wl["monuments"][2018] = {
+    ["az"] = {["start"] = 20180430200000, ["end"] = 20180531195959},
+    ["gw"] = {["start"] = 20180430200000, ["end"] = 20180531195959},
+}
+
+'''
+        result = configuration.parse_config(config)
+        expected = {
+            u'monuments2017': {
+                u'Azerbaijan': {
+                    'start': 20170430200000,
+                    'end': 20170531195959,
+                },
+                u'Guinea-Bissau': {
+                    'start': 20170430200000,
+                    'end': 20170531195959,
+                },
+            },
+            u'monuments2018': {
+                u'Azerbaijan': {
+                    'start': 20180430200000,
+                    'end': 20180531195959,
+                },
+                u'Guinea-Bissau': {
+                    'start': 20180430200000,
+                    'end': 20180531195959,
+                },
+            }
+        }
+        self.assertEquals(result, expected)
+
+
+if __name__ == "__main__":
+    unittest.main()