Skip to content

Commit f97dffd

Browse files
m-aciekm-aciek
authored
Refactor (#34)
* Move template to a separate file * Get number of visitors for only built langs * Raise timeout for getting number of visitors It used to break the CI. * Convert main loop into iterator * Add main block to generate module * Don't link to Plausible when no build * Add mypy * Types improvements * ruff format skip-magic-trailing-comma * reformat * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * improve type * Rename variables * open visitors header link as a blank page
1 parent a2b2b63 commit f97dffd

File tree

7 files changed

+117
-109
lines changed

7 files changed

+117
-109
lines changed

‎.pre-commit-config.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,11 @@ repos:
3434
- id: check-hooks-apply
3535
- id: check-useless-excludes
3636

37+
- repo: https://github.com/pre-commit/mirrors-mypy
38+
rev: v1.14.1
39+
hooks:
40+
- id: mypy
41+
additional_dependencies: [types-docutils, types-polib, types-requests]
42+
3743
ci:
3844
autoupdate_schedule: quarterly

‎generate.py

Lines changed: 41 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,11 @@
99
# ]
1010
# ///
1111
import subprocess
12+
from collections.abc import Iterator
1213
from datetime import datetime, timezone
1314
from pathlib import Path
1415
from tempfile import TemporaryDirectory
16+
from typing import cast
1517

1618
from git import Repo
1719
from jinja2 import Template
@@ -21,106 +23,48 @@
2123
import visitors
2224
from completion import branches_from_devguide, get_completion
2325

24-
completion_progress = []
2526
generation_time = datetime.now(timezone.utc)
2627

27-
with TemporaryDirectory() as clones_dir:
28-
Repo.clone_from(
29-
'https://github.com/python/devguide.git',
30-
devguide_dir := Path(clones_dir, 'devguide'),
31-
depth=1,
32-
)
33-
latest_branch = branches_from_devguide(devguide_dir)[0]
34-
Repo.clone_from(
35-
'https://github.com/python/cpython.git',
36-
Path(clones_dir, 'cpython'),
37-
depth=1,
38-
branch=latest_branch,
39-
)
40-
subprocess.run(['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True)
41-
subprocess.run(
42-
['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True
43-
)
44-
languages_built = dict(build_status.get_languages())
45-
for language, repo in repositories.get_languages_and_repos(devguide_dir):
46-
if repo:
47-
completion_number, translators_number = get_completion(clones_dir, repo)
48-
visitors_number = visitors.get_number_of_visitors(language)
49-
else:
50-
completion_number, translators_number, visitors_number = 0.0, 0, 0
51-
completion_progress.append(
52-
(
53-
language,
54-
repo,
55-
completion_number,
56-
translators_number,
57-
visitors_number,
58-
language in languages_built, # built on docs.python.org
59-
languages_built.get(language), # in switcher
60-
)
28+
29+
def get_completion_progress() -> (
30+
Iterator[tuple[str, str, float, int, int, bool, bool | None]]
31+
):
32+
with TemporaryDirectory() as clones_dir:
33+
Repo.clone_from(
34+
'https://github.com/python/devguide.git',
35+
devguide_dir := Path(clones_dir, 'devguide'),
36+
depth=1,
37+
)
38+
latest_branch = branches_from_devguide(devguide_dir)[0]
39+
Repo.clone_from(
40+
'https://github.com/python/cpython.git',
41+
Path(clones_dir, 'cpython'),
42+
depth=1,
43+
branch=latest_branch,
6144
)
62-
print(completion_progress[-1])
45+
subprocess.run(
46+
['make', '-C', Path(clones_dir, 'cpython/Doc'), 'venv'], check=True
47+
)
48+
subprocess.run(
49+
['make', '-C', Path(clones_dir, 'cpython/Doc'), 'gettext'], check=True
50+
)
51+
languages_built = dict(build_status.get_languages())
52+
for lang, repo in repositories.get_languages_and_repos(devguide_dir):
53+
built = lang in languages_built
54+
in_switcher = languages_built.get(lang)
55+
if not repo:
56+
yield lang, cast(str, repo), 0.0, 0, 0, built, in_switcher
57+
continue
58+
completion, translators = get_completion(clones_dir, repo)
59+
visitors_num = visitors.get_number_of_visitors(lang) if built else 0
60+
yield lang, repo, completion, translators, visitors_num, built, in_switcher
6361

64-
template = Template(
65-
"""
66-
<html lang="en">
67-
<head>
68-
<title>Python Docs Translation Dashboard</title>
69-
<link rel="stylesheet" href="style.css">
70-
</head>
71-
<body>
72-
<h1>Python Docs Translation Dashboard</h1>
73-
<table>
74-
<thead>
75-
<tr>
76-
<th>language</th>
77-
<th>build</th>
78-
<th><a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies">visitors<a/></th>
79-
<th>translators</th>
80-
<th>completion</th>
81-
</tr>
82-
</thead>
83-
<tbody>
84-
{% for language, repo, completion, translators, visitors, build, in_switcher in completion_progress | sort(attribute='2,3') | reverse %}
85-
<tr>
86-
{% if repo %}
87-
<td data-label="language">
88-
<a href="https://github.com/{{ repo }}" target="_blank">
89-
{{ language }}
90-
</a>
91-
</td>
92-
{% else %}
93-
<td data-label="language">{{ language }}</td>
94-
{% endif %}
95-
<td data-label="build">
96-
{% if build %}
97-
<a href="https://docs.python.org/{{ language }}/" target="_blank">✓{% if in_switcher %} in switcher{% endif %}</a>
98-
{% else %}
99-
✗
100-
{% endif %}
101-
</td>
102-
<td data-label="visitors">
103-
<a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ language }}/)))" target="_blank">
104-
{{ '{:,}'.format(visitors) }}
105-
</a>
106-
</td>
107-
<td data-label="translators">{{ '{:,}'.format(translators) }}</td>
108-
<td data-label="completion">
109-
<div class="progress-bar" style="width: {{ completion | round(2) }}%;">{{ completion | round(2) }}%</div>
110-
</td>
111-
</tr>
112-
{% endfor %}
113-
</tbody>
114-
</table>
115-
<p>Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.</p>
116-
</body>
117-
</html>
118-
"""
119-
)
12062

121-
output = template.render(
122-
completion_progress=completion_progress, generation_time=generation_time
123-
)
63+
if __name__ == '__main__':
64+
template = Template(Path('template.html.jinja').read_text())
65+
66+
output = template.render(
67+
completion_progress=get_completion_progress(), generation_time=generation_time
68+
)
12469

125-
with open('index.html', 'w') as file:
126-
file.write(output)
70+
Path('index.html').write_text(output)

‎repositories.py

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,24 @@
1-
import pathlib
21
import re
3-
from typing import Generator, Optional
2+
from collections.abc import Iterator
3+
from pathlib import Path
44

55
from docutils import core
66
from docutils.nodes import table, row
77

88

9-
def get_languages_and_repos(
10-
devguide_dir: pathlib.Path,
11-
) -> Generator[tuple[str, Optional[str]], None, None]:
9+
def get_languages_and_repos(devguide_dir: Path) -> Iterator[tuple[str, str | None]]:
1210
translating = devguide_dir.joinpath('documentation/translating.rst').read_text()
1311
doctree = core.publish_doctree(translating)
1412

1513
for node in doctree.traverse(table):
1614
for row_node in node.traverse(row)[1:]:
1715
language = row_node[0].astext()
1816
repo = row_node[2].astext()
19-
language_code = (
20-
re.match(r'.* \((.*)\)', language).group(1).lower().replace('_', '-')
21-
)
17+
language_match = re.match(r'.* \((.*)\)', language)
18+
if not language_match:
19+
raise ValueError(
20+
f'Expected a language code in brackets in devguide table, found {language}'
21+
)
22+
language_code = language_match.group(1).lower().replace('_', '-')
2223
repo_match = re.match(':github:`GitHub <(.*)>`', repo)
2324
yield language_code, repo_match and repo_match.group(1)

‎ruff.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
[format]
22
quote-style = "single"
3+
skip-magic-trailing-comma = true

‎template.html.jinja

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
<html lang="en">
2+
<head>
3+
<title>Python Docs Translation Dashboard</title>
4+
<link rel="stylesheet" href="style.css">
5+
</head>
6+
<body>
7+
<h1>Python Docs Translation Dashboard</h1>
8+
<table>
9+
<thead>
10+
<tr>
11+
<th>language</th>
12+
<th>build</th>
13+
<th><a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies" target="_blank">visitors</a></th>
14+
<th>translators</th>
15+
<th>completion</th>
16+
</tr>
17+
</thead>
18+
<tbody>
19+
{% for language, repo, completion, translators, visitors, build, in_switcher in completion_progress | sort(attribute='2,3') | reverse %}
20+
<tr>
21+
{% if repo %}
22+
<td data-label="language">
23+
<a href="https://github.com/{{ repo }}" target="_blank">
24+
{{ language }}
25+
</a>
26+
</td>
27+
{% else %}
28+
<td data-label="language">{{ language }}</td>
29+
{% endif %}
30+
<td data-label="build">
31+
{% if build %}
32+
<a href="https://docs.python.org/{{ language }}/" target="_blank">✓{% if in_switcher %} in switcher{% endif %}</a>
33+
{% else %}
34+
✗
35+
{% endif %}
36+
</td>
37+
<td data-label="visitors">
38+
{% if build %}
39+
<a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ language }}/)))" target="_blank">
40+
{{ '{:,}'.format(visitors) }}
41+
</a>
42+
{% else %}
43+
0
44+
{% endif %}
45+
</td>
46+
<td data-label="translators">{{ translators }}</td>
47+
<td data-label="completion">
48+
<div class="progress-bar" style="width: {{ completion | round(2) }}%;">{{ completion | round(2) }}%</div>
49+
</td>
50+
</tr>
51+
{% endfor %}
52+
</tbody>
53+
</table>
54+
<p>Last updated at {{ generation_time.strftime('%A, %d %B %Y, %X %Z') }}.</p>
55+
</body>
56+
</html>

‎translators.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from collections.abc import Generator
1+
from collections.abc import Iterator
22
from pathlib import Path
33

44
from git import Repo
@@ -15,10 +15,10 @@ def get_number_from_git_history(path: Path) -> int:
1515
return len(Repo(path).git.shortlog('-s', 'HEAD').splitlines())
1616

1717

18-
def yield_from_headers(path: Path) -> Generator[str, None, None]:
18+
def yield_from_headers(path: Path) -> Iterator[str]:
1919
for file in path.rglob('*.po'):
2020
try:
21-
header = pofile(file).header.splitlines()
21+
header = pofile(file).header.splitlines() # type: ignore[call-overload] # https://github.com/python/typeshed/pull/13396
2222
except IOError:
2323
continue
2424
if 'Translators:' not in header:

‎visitors.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ def get_number_of_visitors(language: str) -> int:
1010
param = urllib.parse.urlencode(
1111
{'filters': f'[["contains","event:page",["/{language}/"]]]'}
1212
)
13-
r = requests.get(f'https://plausible.io/docs.python.org/export?{param}', timeout=10)
13+
r = requests.get(f'https://plausible.io/docs.python.org/export?{param}', timeout=20)
1414
with (
1515
zipfile.ZipFile(io.BytesIO(r.content), 'r') as z,
1616
z.open('visitors.csv') as csv_file,

0 commit comments

Comments
 (0)