fix: contentlanguages need to be matched before piping output to target

.gitignore

@@ -0,0 +1 @@
+*.json

Dockerfile

@@ -1,8 +1,9 @@
 FROM node:20-alpine
 
-RUN apk add --no-cache jq=1.7.1-r0 curl=8.8.0-r0 && \
+RUN apk add --no-cache curl=8.8.0-r0 python3=3.12.3-r1 && \
 	npm install -g wikibase-cli@18.0.3
 
 COPY --chmod=755 ./transferbot.sh /usr/bin/transferbot
+COPY --chmod=755 ./mangle_data.py /usr/bin/mangle_data
 
 ENTRYPOINT ["transferbot"]

mangle_data.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import fileinput
+from urllib import request
+
+parser = argparse.ArgumentParser(
+    description="Adjust a line from wb-cli so it can be fed into the target wiki"
+)
+
+parser.add_argument("-t", "--target", action="store", dest="target", required=True)
+parser.add_argument("-p", "--pick", action="append", dest="pick", required=True)
+
+
+def process_line(line, pick=[], select_languages=set()):
+    data = json.loads(line)
+    out = {}
+    for key, value in data.items():
+        if key not in pick:
+            continue
+
+        if not isinstance(value, dict):
+            out[key] = value
+            continue
+
+        out[key] = {}
+        for lang, value in data[key].items():
+            if lang in select_languages:
+                out[key][lang] = value
+
+    return out
+
+
+def get_contentlanguages(target_origin):
+    with request.urlopen(
+        f"{target_origin}/w/api.php?action=query&meta=wbcontentlanguages&format=json"
+    ) as response:
+        raw_body = response.read()
+    body = json.loads(raw_body)
+    return set(body["query"]["wbcontentlanguages"].keys())
+
+
+def main():
+    args = parser.parse_args()
+    target_languages = get_contentlanguages(args.target)
+
+    for line in fileinput.input("-"):
+        out = process_line(line, pick=args.pick, select_languages=target_languages)
+        print(json.dumps(out, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

transferbot.sh

@@ -37,5 +37,5 @@ CREDS
 } > $(wb config path)
 
 wb data $@ --instance "$source_wiki_origin" |\
-  jq --compact-output '{type,labels,descriptions,aliases,datatype}' |\
+  mangle_data -t "$target_wiki_origin" -p type -p labels -p descriptions -p aliases -p datatype |\
   wb create-entity --batch --instance "$target_wiki_origin"