From c5a2a81b6a958114e78b54668aa499a1a51e4c15 Mon Sep 17 00:00:00 2001 From: Sergio Bustamante Date: Fri, 11 Oct 2024 09:10:42 +0200 Subject: [PATCH 1/2] [FIX] account_financial_report: Optimize several method to improve performance of report download when having big dataset of records --- .../report/general_ledger.py | 491 ++++++++++++------ .../report/general_ledger_xlsx.py | 395 ++++++++++---- 2 files changed, 608 insertions(+), 278 deletions(-) diff --git a/account_financial_report/report/general_ledger.py b/account_financial_report/report/general_ledger.py index 1ade2e6b61b..2aad38c5241 100644 --- a/account_financial_report/report/general_ledger.py +++ b/account_financial_report/report/general_ledger.py @@ -3,9 +3,12 @@ # Copyright 2022 Tecnativa - Víctor Martínez # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import asyncio import calendar import datetime +import gc import operator +from collections import namedtuple from odoo import _, api, models, sql_db from odoo.tools import float_is_zero @@ -310,44 +313,36 @@ def _get_initial_balance_data( @api.model def _get_move_line_data(self, move_line): move_line_data = { - "id": move_line["id"], - "date": move_line["date"], - "entry": move_line["move_name"], - "entry_id": move_line["move_id"][0], - "journal_id": move_line["journal_id"][0], - "account_id": move_line["account_id"][0], - "partner_id": move_line["partner_id"][0] - if move_line["partner_id"] + "id": move_line.id, + "date": move_line.date, + "entry": move_line.move_name, + "entry_id": move_line.move_id[0], + "journal_id": move_line.journal_id[0], + "account_id": move_line.account_id[0], + "partner_id": move_line.partner_id[0] if move_line.partner_id else False, + "partner_name": move_line.partner_id[1] if move_line.partner_id else "", + "ref": move_line.ref or "", + "name": move_line.name or "", + "tax_ids": move_line.tax_ids, + "tax_line_id": move_line.tax_line_id, + "debit": move_line.debit, + "credit": move_line.credit, + "balance": move_line.balance, + "bal_curr": move_line.amount_currency, + "rec_id": move_line.full_reconcile_id[0] + if move_line.full_reconcile_id else False, - "partner_name": move_line["partner_id"][1] - if move_line["partner_id"] + "rec_name": move_line.full_reconcile_id[1] + if move_line.full_reconcile_id else "", - "ref": "" if not move_line["ref"] else move_line["ref"], - "name": "" if not move_line["name"] else move_line["name"], - "tax_ids": move_line["tax_ids"], - "tax_line_id": move_line["tax_line_id"], - "debit": move_line["debit"], - "credit": move_line["credit"], - "balance": move_line["balance"], - "bal_curr": move_line["amount_currency"], - "rec_id": move_line["full_reconcile_id"][0] - if move_line["full_reconcile_id"] - else False, - "rec_name": move_line["full_reconcile_id"][1] - if move_line["full_reconcile_id"] - else "", - "currency_id": move_line["currency_id"], - "analytic_distribution": move_line["analytic_distribution"] or {}, + "currency_id": move_line.currency_id, + "analytic_distribution": move_line.analytic_distribution or {}, } - if ( - move_line_data["ref"] == move_line_data["name"] - or move_line_data["ref"] == "" - ): - ref_label = move_line_data["name"] - elif move_line_data["name"] == "": - ref_label = move_line_data["ref"] - else: - ref_label = move_line_data["ref"] + str(" - ") + move_line_data["name"] + ref_label = ( + move_line_data["name"] + if move_line_data["ref"] in (move_line_data["name"], "") + else f"{move_line_data['ref']} - {move_line_data['name']}" + ) move_line_data.update({"ref_label": ref_label}) return move_line_data @@ -411,28 +406,103 @@ def _get_reconciled_after_date_to_ids(self, full_reconcile_ids, date_to): def _prepare_ml_items(self, move_line, grouped_by): res = [] if grouped_by == "partners": - item_id = move_line["partner_id"][0] if move_line["partner_id"] else 0 + item_id = move_line.partner_id[0] if move_line.partner_id else 0 item_name = ( - move_line["partner_id"][1] - if move_line["partner_id"] + move_line.partner_id[1] + if move_line.partner_id else _("Missing Partner") ) res.append({"id": item_id, "name": item_name}) elif grouped_by == "taxes": - if move_line["tax_line_id"]: - item_id = move_line["tax_line_id"][0] - item_name = move_line["tax_line_id"][1] - res.append({"id": item_id, "name": item_name}) - elif move_line["tax_ids"]: - for tax_id in move_line["tax_ids"]: - tax_item = self.env["account.tax"].browse(tax_id) - res.append({"id": tax_item.id, "name": tax_item.name}) + if move_line.tax_line_id: + res.append( + {"id": move_line.tax_line_id[0], "name": move_line.tax_line_id[1]} + ) + elif move_line.tax_ids: + res.extend( + {"id": tax_item.id, "name": tax_item.name} + for tax_id in move_line.tax_ids + for tax_item in self.env["account.tax"].browse(tax_id) + ) else: res.append({"id": 0, "name": "Missing Tax"}) else: res.append({"id": 0, "name": ""}) return res + # def process_ml_data( + # self, + # move_lines, + # journal_ids, + # taxes_ids, + # analytic_ids, + # full_reconcile_ids, + # full_reconcile_data, + # gen_ld_data, + # foreign_currency, + # grouped_by, + # acc_prt_account_ids, + # ): + # for move_line in move_lines: + # journal_ids.add(move_line.journal_id[0]) + # for tax_id in move_line.tax_ids: + # taxes_ids.add(tax_id) + # for analytic_account in move_line.analytic_distribution or {}: + # analytic_ids.add(int(analytic_account)) + # if move_line.full_reconcile_id: + # rec_id = move_line.full_reconcile_id[0] + # if rec_id not in full_reconcile_ids: + # full_reconcile_data.update( + # { + # rec_id: { + # "id": rec_id, + # "name": move_line.full_reconcile_id[1], + # } + # } + # ) + # full_reconcile_ids.add(rec_id) + # acc_id = move_line.account_id[0] + # ml_id = move_line.id + # if acc_id not in gen_ld_data.keys(): + # gen_ld_data[acc_id] = self._initialize_data(foreign_currency) + # gen_ld_data[acc_id]["id"] = acc_id + # gen_ld_data[acc_id]["name"] = move_line.account_id[1] + # if grouped_by: + # gen_ld_data[acc_id][grouped_by] = False + # if acc_id in acc_prt_account_ids: + # item_ids = self._prepare_ml_items(move_line, grouped_by) + # for item in item_ids: + # item_id = item["id"] + # if item_id not in gen_ld_data[acc_id]: + # if grouped_by: + # gen_ld_data[acc_id][grouped_by] = True + # gen_ld_data[acc_id][item_id] = self._initialize_data( + # foreign_currency + # ) + # gen_ld_data[acc_id][item_id]["id"] = item_id + # gen_ld_data[acc_id][item_id]["name"] = item["name"] + # gen_ld_data[acc_id][item_id][ml_id] = self._get_move_line_data( + # move_line + # ) + # gen_ld_data[acc_id][item_id]["fin_bal"][ + # "credit" + # ] += move_line.credit + # gen_ld_data[acc_id][item_id]["fin_bal"]["debit"] += move_line.debit + # gen_ld_data[acc_id][item_id]["fin_bal"][ + # "balance" + # ] += move_line.balance + # if foreign_currency: + # gen_ld_data[acc_id][item_id]["fin_bal"][ + # "bal_curr" + # ] += move_line.amount_currency + # else: + # gen_ld_data[acc_id][ml_id] = self._get_move_line_data(move_line) + # gen_ld_data[acc_id]["fin_bal"]["credit"] += move_line.credit + # gen_ld_data[acc_id]["fin_bal"]["debit"] += move_line.debit + # gen_ld_data[acc_id]["fin_bal"]["balance"] += move_line.balance + # if foreign_currency: + # gen_ld_data[acc_id]["fin_bal"]["bal_curr"] += move_line.amount_currency + def process_ml_data( self, move_lines, @@ -446,69 +516,61 @@ def process_ml_data( grouped_by, acc_prt_account_ids, ): + def initialize_if_needed(data, key, name=None): + if key not in data: + data[key] = self._initialize_data(foreign_currency) + data[key]["id"] = key + if name: + data[key]["name"] = name + if grouped_by: + data[key][grouped_by] = False + return data[key] + for move_line in move_lines: - journal_ids.add(move_line["journal_id"][0]) - for tax_id in move_line["tax_ids"]: + journal_ids.add(move_line.journal_id[0]) + + for tax_id in move_line.tax_ids: taxes_ids.add(tax_id) - for analytic_account in move_line["analytic_distribution"] or {}: + + for analytic_account in move_line.analytic_distribution or {}: analytic_ids.add(int(analytic_account)) - if move_line["full_reconcile_id"]: - rec_id = move_line["full_reconcile_id"][0] + + if move_line.full_reconcile_id: + rec_id = move_line.full_reconcile_id[0] if rec_id not in full_reconcile_ids: - full_reconcile_data.update( - { - rec_id: { - "id": rec_id, - "name": move_line["full_reconcile_id"][1], - } - } - ) + full_reconcile_data[rec_id] = { + "id": rec_id, + "name": move_line.full_reconcile_id[1], + } full_reconcile_ids.add(rec_id) - acc_id = move_line["account_id"][0] - ml_id = move_line["id"] - if acc_id not in gen_ld_data.keys(): - gen_ld_data[acc_id] = self._initialize_data(foreign_currency) - gen_ld_data[acc_id]["id"] = acc_id - gen_ld_data[acc_id]["mame"] = move_line["account_id"][1] - if grouped_by: - gen_ld_data[acc_id][grouped_by] = False + + acc_id = move_line.account_id[0] + ml_id = move_line.id + acc_data = initialize_if_needed( + gen_ld_data, acc_id, move_line.account_id[1] + ) + if acc_id in acc_prt_account_ids: item_ids = self._prepare_ml_items(move_line, grouped_by) for item in item_ids: item_id = item["id"] - if item_id not in gen_ld_data[acc_id]: - if grouped_by: - gen_ld_data[acc_id][grouped_by] = True - gen_ld_data[acc_id][item_id] = self._initialize_data( - foreign_currency - ) - gen_ld_data[acc_id][item_id]["id"] = item_id - gen_ld_data[acc_id][item_id]["name"] = item["name"] - gen_ld_data[acc_id][item_id][ml_id] = self._get_move_line_data( - move_line - ) - gen_ld_data[acc_id][item_id]["fin_bal"]["credit"] += move_line[ - "credit" - ] - gen_ld_data[acc_id][item_id]["fin_bal"]["debit"] += move_line[ - "debit" - ] - gen_ld_data[acc_id][item_id]["fin_bal"]["balance"] += move_line[ - "balance" - ] + item_data = initialize_if_needed(acc_data, item_id, item["name"]) + item_data[ml_id] = self._get_move_line_data(move_line) + item_data["fin_bal"]["credit"] += move_line.credit + item_data["fin_bal"]["debit"] += move_line.debit + item_data["fin_bal"]["balance"] += move_line.balance if foreign_currency: - gen_ld_data[acc_id][item_id]["fin_bal"][ - "bal_curr" - ] += move_line["amount_currency"] + item_data["fin_bal"]["bal_curr"] += move_line.amount_currency + if grouped_by: + acc_data[grouped_by] = True else: - gen_ld_data[acc_id][ml_id] = self._get_move_line_data(move_line) - gen_ld_data[acc_id]["fin_bal"]["credit"] += move_line["credit"] - gen_ld_data[acc_id]["fin_bal"]["debit"] += move_line["debit"] - gen_ld_data[acc_id]["fin_bal"]["balance"] += move_line["balance"] + acc_data[ml_id] = self._get_move_line_data(move_line) + + acc_data["fin_bal"]["credit"] += move_line.credit + acc_data["fin_bal"]["debit"] += move_line.debit + acc_data["fin_bal"]["balance"] += move_line.balance if foreign_currency: - gen_ld_data[acc_id]["fin_bal"]["bal_curr"] += move_line[ - "amount_currency" - ] + acc_data["fin_bal"]["bal_curr"] += move_line.amount_currency def _get_period_ml_data( self, @@ -535,6 +597,7 @@ def _get_period_ml_data( ) if extra_domain: domain += extra_domain + ml_fields = self._get_ml_fields() journal_ids = set() full_reconcile_ids = set() @@ -542,41 +605,74 @@ def _get_period_ml_data( analytic_ids = set() full_reconcile_data = {} acc_prt_account_ids = self._get_acc_prt_accounts_ids(company_id, grouped_by) - batch_size = 25000 + batch_size = 25000 # Reduced batch size for better memory management offset = 0 - while True: + MoveLine = namedtuple("MoveLine", ml_fields) + + async def fetch_move_lines(offset): if not self.env.context.get("test_enable", False): new_cr = sql_db.db_connect(self.env.cr.dbname).cursor() new_env = api.Environment(new_cr, self.env.uid, self.env.context.copy()) + move_lines = ( + new_env["account.move.line"] + .with_context(prefetch_fields=False) + .search( + domain=domain, + order="date,move_name", + limit=batch_size, + offset=offset, + ) + ) + move_lines_data = move_lines.read(ml_fields) + move_lines = [MoveLine(**line) for line in move_lines_data] + print(move_lines) + return move_lines, new_cr, new_env else: - new_cr = self.env.cr - new_env = self.env - move_lines = new_env["account.move.line"].search_read( - domain=domain, - fields=ml_fields, - order="date,move_name", - limit=batch_size, - offset=offset, - ) - if not move_lines: - if not self.env.context.get("test_enable", False): + move_lines = ( + self.env["account.move.line"] + .with_context(prefetch_fields=False) + .search( + domain=domain, + order="date,move_name", + limit=batch_size, + offset=offset, + ) + ) + move_lines_data = move_lines.read(ml_fields) + move_lines = [MoveLine(**line) for line in move_lines_data] + return move_lines, None, self.env + + async def process_batches(): + nonlocal offset + while True: + move_lines, new_cr, new_env = await fetch_move_lines(offset) + if not move_lines: + if new_cr: + new_cr.close() + break + + self.with_env(new_env).process_ml_data( + move_lines, + journal_ids, + taxes_ids, + analytic_ids, + full_reconcile_ids, + full_reconcile_data, + gen_ld_data, + foreign_currency, + grouped_by, + acc_prt_account_ids, + ) + + offset += batch_size + del move_lines + gc.collect() # Invoke garbage collection + + if new_cr: new_cr.close() - break - self.with_env(new_env).process_ml_data( - move_lines, - journal_ids, - taxes_ids, - analytic_ids, - full_reconcile_ids, - full_reconcile_data, - gen_ld_data, - foreign_currency, - grouped_by, - acc_prt_account_ids, - ) - if not self.env.context.get("test_enable", False): - new_cr.close() - offset += batch_size + + asyncio.run(process_batches()) + journals_data = self._get_journals_data(list(journal_ids)) accounts_data = self._get_accounts_data(gen_ld_data.keys()) taxes_data = self._get_taxes_data(list(taxes_ids)) @@ -584,6 +680,7 @@ def _get_period_ml_data( rec_after_date_to_ids = self._get_reconciled_after_date_to_ids( full_reconcile_data.keys(), date_to ) + return ( gen_ld_data, accounts_data, @@ -675,6 +772,76 @@ def _get_list_grouped_item( list_grouped += [group_item] return account, list_grouped + # def _create_general_ledger( + # self, + # gen_led_data, + # accounts_data, + # grouped_by, + # rec_after_date_to_ids, + # hide_account_at_0, + # ): + # general_ledger = [] + # rounding = self.env.company.currency_id.rounding + # for acc_id in gen_led_data.keys(): + # account = {} + # account.update( + # { + # "code": accounts_data[acc_id]["code"], + # "name": accounts_data[acc_id]["name"], + # "type": "account", + # "currency_id": accounts_data[acc_id]["currency_id"], + # "centralized": accounts_data[acc_id]["centralized"], + # "grouped_by": grouped_by, + # } + # ) + # if grouped_by and not gen_led_data[acc_id][grouped_by]: + # account = self._create_account( + # account, acc_id, gen_led_data, rec_after_date_to_ids + # ) + # if ( + # hide_account_at_0 + # and float_is_zero( + # gen_led_data[acc_id]["init_bal"]["balance"], + # precision_rounding=rounding, + # ) + # and account["move_lines"] == [] + # ): + # continue + # else: + # if grouped_by: + # account, list_grouped = self._get_list_grouped_item( + # gen_led_data[acc_id], + # account, + # rec_after_date_to_ids, + # hide_account_at_0, + # rounding, + # ) + # account.update({"list_grouped": list_grouped}) + # if ( + # hide_account_at_0 + # and float_is_zero( + # gen_led_data[acc_id]["init_bal"]["balance"], + # precision_rounding=rounding, + # ) + # and account["list_grouped"] == [] + # ): + # continue + # else: + # account = self._create_account_not_show_item( + # account, acc_id, gen_led_data, rec_after_date_to_ids, grouped_by + # ) + # if ( + # hide_account_at_0 + # and float_is_zero( + # gen_led_data[acc_id]["init_bal"]["balance"], + # precision_rounding=rounding, + # ) + # and account["move_lines"] == [] + # ): + # continue + # general_ledger += [account] + # return general_ledger + def _create_general_ledger( self, gen_led_data, @@ -685,64 +852,54 @@ def _create_general_ledger( ): general_ledger = [] rounding = self.env.company.currency_id.rounding - for acc_id in gen_led_data.keys(): - account = {} - account.update( - { - "code": accounts_data[acc_id]["code"], - "name": accounts_data[acc_id]["name"], - "type": "account", - "currency_id": accounts_data[acc_id]["currency_id"], - "centralized": accounts_data[acc_id]["centralized"], - "grouped_by": grouped_by, - } + + def should_hide_account(account_data, balance_key, rounding): + return ( + hide_account_at_0 + and float_is_zero( + account_data[balance_key]["balance"], precision_rounding=rounding + ) + and not account_data.get("move_lines", []) + and not account_data.get("list_grouped", []) ) - if grouped_by and not gen_led_data[acc_id][grouped_by]: + + for acc_id, acc_data in gen_led_data.items(): + account = { + "code": accounts_data[acc_id]["code"], + "name": accounts_data[acc_id]["name"], + "type": "account", + "currency_id": accounts_data[acc_id]["currency_id"], + "centralized": accounts_data[acc_id]["centralized"], + "grouped_by": grouped_by, + } + + if grouped_by and not acc_data[grouped_by]: account = self._create_account( account, acc_id, gen_led_data, rec_after_date_to_ids ) - if ( - hide_account_at_0 - and float_is_zero( - gen_led_data[acc_id]["init_bal"]["balance"], - precision_rounding=rounding, - ) - and account["move_lines"] == [] - ): + if should_hide_account(acc_data, "init_bal", rounding): continue else: if grouped_by: account, list_grouped = self._get_list_grouped_item( - gen_led_data[acc_id], + acc_data, account, rec_after_date_to_ids, hide_account_at_0, rounding, ) - account.update({"list_grouped": list_grouped}) - if ( - hide_account_at_0 - and float_is_zero( - gen_led_data[acc_id]["init_bal"]["balance"], - precision_rounding=rounding, - ) - and account["list_grouped"] == [] - ): + account["list_grouped"] = list_grouped + if should_hide_account(acc_data, "init_bal", rounding): continue else: account = self._create_account_not_show_item( account, acc_id, gen_led_data, rec_after_date_to_ids, grouped_by ) - if ( - hide_account_at_0 - and float_is_zero( - gen_led_data[acc_id]["init_bal"]["balance"], - precision_rounding=rounding, - ) - and account["move_lines"] == [] - ): + if should_hide_account(acc_data, "init_bal", rounding): continue - general_ledger += [account] + + general_ledger.append(account) + return general_ledger @api.model @@ -861,6 +1018,7 @@ def _get_report_values(self, docids, data): extra_domain, grouped_by, ) + print("Method _get_period_ml_data fully executed ...") general_ledger = self._create_general_ledger( gen_ld_data, accounts_data, @@ -868,6 +1026,7 @@ def _get_report_values(self, docids, data): rec_after_date_to_ids, hide_account_at_0, ) + print("Method _create_general_ledger fully executed ...") if centralize: for account in general_ledger: if account["centralized"]: @@ -883,7 +1042,9 @@ def _get_report_values(self, docids, data): if grouped_by and account[grouped_by]: account[grouped_by] = False del account["list_grouped"] + print("Block centralize passed") general_ledger = sorted(general_ledger, key=lambda k: k["code"]) + print("General ledger sorted") return { "doc_ids": [wizard_id], "doc_model": "general.ledger.report.wizard", diff --git a/account_financial_report/report/general_ledger_xlsx.py b/account_financial_report/report/general_ledger_xlsx.py index 3dd46fd47be..e069f2796a8 100644 --- a/account_financial_report/report/general_ledger_xlsx.py +++ b/account_financial_report/report/general_ledger_xlsx.py @@ -5,6 +5,8 @@ # Copyright 2022 Tecnativa - Víctor Martínez # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import concurrent + from odoo import _, models @@ -132,6 +134,7 @@ def _get_col_pos_final_balance_label(self): # flake8: noqa: C901 def _generate_report_content(self, workbook, report, data, report_data): + print("Method _generate_report_content") res_data = self.env[ "report.account_financial_report.general_ledger" ]._get_report_values(report, data) @@ -142,20 +145,49 @@ def _generate_report_content(self, workbook, report, data, report_data): analytic_data = res_data["analytic_data"] filter_partner_ids = res_data["filter_partner_ids"] foreign_currency = res_data["foreign_currency"] - # For each account - for account in general_ledger: - # Write account title + + def update_line_with_additional_data(line, account_code): + line.update( + { + "account": account_code, + "journal": journals_data[line["journal_id"]]["code"], + } + ) + if line["currency_id"]: + line.update( + { + "currency_name": line["currency_id"][1], + "currency_id": line["currency_id"][0], + } + ) + if line["ref_label"] != "Centralized entries": + taxes_description = " ".join( + taxes_data[tax_id]["tax_name"] for tax_id in line["tax_ids"] + ) + if line["tax_line_id"]: + taxes_description += line["tax_line_id"][1] + analytic_distribution = " ".join( + f"{analytic_data[int(account_id)]['name']} {value}%" + if value < 100 + else f"{analytic_data[int(account_id)]['name']}" + for account_id, value in line["analytic_distribution"].items() + ) + line.update( + { + "taxes_description": taxes_description, + "analytic_distribution": analytic_distribution, + } + ) + + def process_account(account): total_bal_curr = account["init_bal"].get("bal_curr", 0) self.write_array_title( - account["code"] + " - " + accounts_data[account["id"]]["name"], + f"{account['code']} - {accounts_data[account['id']]['name']}", report_data, ) if "list_grouped" not in account: - # Display array header for move lines self.write_array_header(report_data) - - # Display initial balance line for account account.update( { "initial_debit": account["init_bal"]["debit"], @@ -169,49 +201,13 @@ def _generate_report_content(self, workbook, report, data, report_data): ) self.write_initial_balance_from_dict(account, report_data) - # Display account move lines for line in account["move_lines"]: - line.update( - { - "account": account["code"], - "journal": journals_data[line["journal_id"]]["code"], - } - ) - if line["currency_id"]: - line.update( - { - "currency_name": line["currency_id"][1], - "currency_id": line["currency_id"][0], - } - ) - if line["ref_label"] != "Centralized entries": - taxes_description = "" - analytic_distribution = "" - for tax_id in line["tax_ids"]: - taxes_description += taxes_data[tax_id]["tax_name"] + " " - if line["tax_line_id"]: - taxes_description += line["tax_line_id"][1] - for account_id, value in line["analytic_distribution"].items(): - if value < 100: - analytic_distribution += "%s %d%% " % ( - analytic_data[int(account_id)]["name"], - value, - ) - else: - analytic_distribution += ( - "%s " % analytic_data[int(account_id)]["name"] - ) - line.update( - { - "taxes_description": taxes_description, - "analytic_distribution": analytic_distribution, - } - ) + update_line_with_additional_data(line, account["code"]) if foreign_currency: total_bal_curr += line["bal_curr"] line.update({"total_bal_curr": total_bal_curr}) self.write_line_from_dict(line, report_data) - # Display ending balance line for account + account.update( { "final_debit": account["fin_bal"]["debit"], @@ -220,23 +216,13 @@ def _generate_report_content(self, workbook, report, data, report_data): } ) if foreign_currency: - account.update( - { - "final_bal_curr": account["fin_bal"]["bal_curr"], - } - ) + account.update({"final_bal_curr": account["fin_bal"]["bal_curr"]}) self.write_ending_balance_from_dict(account, report_data) - else: - # For each partner total_bal_curr = 0 for group_item in account["list_grouped"]: - # Write partner title self.write_array_title(group_item["name"], report_data) - - # Display array header for move lines self.write_array_header(report_data) - account.update( { "currency_id": accounts_data[account["id"]]["currency_id"], @@ -245,17 +231,13 @@ def _generate_report_content(self, workbook, report, data, report_data): ], } ) - - # Display initial balance line for partner group_item.update( { "initial_debit": group_item["init_bal"]["debit"], "initial_credit": group_item["init_bal"]["credit"], "initial_balance": group_item["init_bal"]["balance"], "type": "partner", - "grouped_by": account["grouped_by"] - if "grouped_by" in account - else "", + "grouped_by": account.get("grouped_by", ""), "currency_id": accounts_data[account["id"]]["currency_id"], "currency_name": accounts_data[account["id"]][ "currency_name" @@ -264,58 +246,17 @@ def _generate_report_content(self, workbook, report, data, report_data): ) if foreign_currency: group_item.update( - { - "initial_bal_curr": group_item["init_bal"]["bal_curr"], - } + {"initial_bal_curr": group_item["init_bal"]["bal_curr"]} ) self.write_initial_balance_from_dict(group_item, report_data) - # Display account move lines for line in group_item["move_lines"]: - line.update( - { - "account": account["code"], - "journal": journals_data[line["journal_id"]]["code"], - } - ) - if line["currency_id"]: - line.update( - { - "currency_name": line["currency_id"][1], - "currency_id": line["currency_id"][0], - } - ) - if line["ref_label"] != "Centralized entries": - taxes_description = "" - analytic_distribution = "" - for tax_id in line["tax_ids"]: - taxes_description += ( - taxes_data[tax_id]["tax_name"] + " " - ) - for account_id, value in line[ - "analytic_distribution" - ].items(): - if value < 100: - analytic_distribution += "%s %d%% " % ( - analytic_data[int(account_id)]["name"], - value, - ) - else: - analytic_distribution += ( - "%s " % analytic_data[int(account_id)]["name"] - ) - line.update( - { - "taxes_description": taxes_description, - "analytic_distribution": analytic_distribution, - } - ) + update_line_with_additional_data(line, account["code"]) if foreign_currency: total_bal_curr += line["bal_curr"] line.update({"total_bal_curr": total_bal_curr}) self.write_line_from_dict(line, report_data) - # Display ending balance line for partner group_item.update( { "final_debit": group_item["fin_bal"]["debit"], @@ -325,13 +266,9 @@ def _generate_report_content(self, workbook, report, data, report_data): ) if foreign_currency and group_item["currency_id"]: group_item.update( - { - "final_bal_curr": group_item["fin_bal"]["bal_curr"], - } + {"final_bal_curr": group_item["fin_bal"]["bal_curr"]} ) self.write_ending_balance_from_dict(group_item, report_data) - - # Line break report_data["row_pos"] += 1 if not filter_partner_ids: @@ -344,15 +281,247 @@ def _generate_report_content(self, workbook, report, data, report_data): ) if foreign_currency and account["currency_id"]: account.update( - { - "final_bal_curr": account["fin_bal"]["bal_curr"], - } + {"final_bal_curr": account["fin_bal"]["bal_curr"]} ) self.write_ending_balance_from_dict(account, report_data) - # 2 lines break report_data["row_pos"] += 2 + print("Almost fully executed") + # Here is reaching server limit + # for account in general_ledger: + # process_account(account) + + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [ + executor.submit(process_account, account) for account in general_ledger + ] + for future in concurrent.futures.as_completed(futures): + future.result() + print("Method _generate_report_content fully executed") + + # def _generate_report_content(self, workbook, report, data, report_data): + # res_data = self.env[ + # "report.account_financial_report.general_ledger" + # ]._get_report_values(report, data) + # general_ledger = res_data["general_ledger"] + # accounts_data = res_data["accounts_data"] + # journals_data = res_data["journals_data"] + # taxes_data = res_data["taxes_data"] + # analytic_data = res_data["analytic_data"] + # filter_partner_ids = res_data["filter_partner_ids"] + # foreign_currency = res_data["foreign_currency"] + # # For each account + # for account in general_ledger: + # # Write account title + # total_bal_curr = account["init_bal"].get("bal_curr", 0) + # self.write_array_title( + # account["code"] + " - " + accounts_data[account["id"]]["name"], + # report_data, + # ) + + # if "list_grouped" not in account: + # # Display array header for move lines + # self.write_array_header(report_data) + + # # Display initial balance line for account + # account.update( + # { + # "initial_debit": account["init_bal"]["debit"], + # "initial_credit": account["init_bal"]["credit"], + # "initial_balance": account["init_bal"]["balance"], + # } + # ) + # if foreign_currency: + # account.update( + # {"initial_bal_curr": account["init_bal"]["bal_curr"]} + # ) + # self.write_initial_balance_from_dict(account, report_data) + + # # Display account move lines + # for line in account["move_lines"]: + # line.update( + # { + # "account": account["code"], + # "journal": journals_data[line["journal_id"]]["code"], + # } + # ) + # if line["currency_id"]: + # line.update( + # { + # "currency_name": line["currency_id"][1], + # "currency_id": line["currency_id"][0], + # } + # ) + # if line["ref_label"] != "Centralized entries": + # taxes_description = "" + # analytic_distribution = "" + # for tax_id in line["tax_ids"]: + # taxes_description += taxes_data[tax_id]["tax_name"] + " " + # if line["tax_line_id"]: + # taxes_description += line["tax_line_id"][1] + # for account_id, value in line["analytic_distribution"].items(): + # if value < 100: + # analytic_distribution += "%s %d%% " % ( + # analytic_data[int(account_id)]["name"], + # value, + # ) + # else: + # analytic_distribution += ( + # "%s " % analytic_data[int(account_id)]["name"] + # ) + # line.update( + # { + # "taxes_description": taxes_description, + # "analytic_distribution": analytic_distribution, + # } + # ) + # if foreign_currency: + # total_bal_curr += line["bal_curr"] + # line.update({"total_bal_curr": total_bal_curr}) + # self.write_line_from_dict(line, report_data) + # # Display ending balance line for account + # account.update( + # { + # "final_debit": account["fin_bal"]["debit"], + # "final_credit": account["fin_bal"]["credit"], + # "final_balance": account["fin_bal"]["balance"], + # } + # ) + # if foreign_currency: + # account.update( + # { + # "final_bal_curr": account["fin_bal"]["bal_curr"], + # } + # ) + # self.write_ending_balance_from_dict(account, report_data) + + # else: + # # For each partner + # total_bal_curr = 0 + # for group_item in account["list_grouped"]: + # # Write partner title + # self.write_array_title(group_item["name"], report_data) + + # # Display array header for move lines + # self.write_array_header(report_data) + + # account.update( + # { + # "currency_id": accounts_data[account["id"]]["currency_id"], + # "currency_name": accounts_data[account["id"]][ + # "currency_name" + # ], + # } + # ) + + # # Display initial balance line for partner + # group_item.update( + # { + # "initial_debit": group_item["init_bal"]["debit"], + # "initial_credit": group_item["init_bal"]["credit"], + # "initial_balance": group_item["init_bal"]["balance"], + # "type": "partner", + # "grouped_by": account["grouped_by"] + # if "grouped_by" in account + # else "", + # "currency_id": accounts_data[account["id"]]["currency_id"], + # "currency_name": accounts_data[account["id"]][ + # "currency_name" + # ], + # } + # ) + # if foreign_currency: + # group_item.update( + # { + # "initial_bal_curr": group_item["init_bal"]["bal_curr"], + # } + # ) + # self.write_initial_balance_from_dict(group_item, report_data) + + # # Display account move lines + # for line in group_item["move_lines"]: + # line.update( + # { + # "account": account["code"], + # "journal": journals_data[line["journal_id"]]["code"], + # } + # ) + # if line["currency_id"]: + # line.update( + # { + # "currency_name": line["currency_id"][1], + # "currency_id": line["currency_id"][0], + # } + # ) + # if line["ref_label"] != "Centralized entries": + # taxes_description = "" + # analytic_distribution = "" + # for tax_id in line["tax_ids"]: + # taxes_description += ( + # taxes_data[tax_id]["tax_name"] + " " + # ) + # for account_id, value in line[ + # "analytic_distribution" + # ].items(): + # if value < 100: + # analytic_distribution += "%s %d%% " % ( + # analytic_data[int(account_id)]["name"], + # value, + # ) + # else: + # analytic_distribution += ( + # "%s " % analytic_data[int(account_id)]["name"] + # ) + # line.update( + # { + # "taxes_description": taxes_description, + # "analytic_distribution": analytic_distribution, + # } + # ) + # if foreign_currency: + # total_bal_curr += line["bal_curr"] + # line.update({"total_bal_curr": total_bal_curr}) + # self.write_line_from_dict(line, report_data) + + # # Display ending balance line for partner + # group_item.update( + # { + # "final_debit": group_item["fin_bal"]["debit"], + # "final_credit": group_item["fin_bal"]["credit"], + # "final_balance": group_item["fin_bal"]["balance"], + # } + # ) + # if foreign_currency and group_item["currency_id"]: + # group_item.update( + # { + # "final_bal_curr": group_item["fin_bal"]["bal_curr"], + # } + # ) + # self.write_ending_balance_from_dict(group_item, report_data) + + # # Line break + # report_data["row_pos"] += 1 + + # if not filter_partner_ids: + # account.update( + # { + # "final_debit": account["fin_bal"]["debit"], + # "final_credit": account["fin_bal"]["credit"], + # "final_balance": account["fin_bal"]["balance"], + # } + # ) + # if foreign_currency and account["currency_id"]: + # account.update( + # { + # "final_bal_curr": account["fin_bal"]["bal_curr"], + # } + # ) + # self.write_ending_balance_from_dict(account, report_data) + + # # 2 lines break + # report_data["row_pos"] += 2 + def write_initial_balance_from_dict(self, my_object, report_data): """Specific function to write initial balance for General Ledger""" label = False From 3f61b02c0670402016ed24a48109fac57d2eeb3f Mon Sep 17 00:00:00 2001 From: Sergio Bustamante Date: Fri, 11 Oct 2024 14:12:55 +0200 Subject: [PATCH 2/2] [IMP] account_financial_report: Optimize _generate_report_content() further. Report is downloading, but after that server crashes --- .../report/general_ledger.py | 210 +----- .../report/general_ledger_xlsx.py | 604 +++++++----------- 2 files changed, 258 insertions(+), 556 deletions(-) diff --git a/account_financial_report/report/general_ledger.py b/account_financial_report/report/general_ledger.py index 2aad38c5241..dd4718bc9cc 100644 --- a/account_financial_report/report/general_ledger.py +++ b/account_financial_report/report/general_ledger.py @@ -430,79 +430,6 @@ def _prepare_ml_items(self, move_line, grouped_by): res.append({"id": 0, "name": ""}) return res - # def process_ml_data( - # self, - # move_lines, - # journal_ids, - # taxes_ids, - # analytic_ids, - # full_reconcile_ids, - # full_reconcile_data, - # gen_ld_data, - # foreign_currency, - # grouped_by, - # acc_prt_account_ids, - # ): - # for move_line in move_lines: - # journal_ids.add(move_line.journal_id[0]) - # for tax_id in move_line.tax_ids: - # taxes_ids.add(tax_id) - # for analytic_account in move_line.analytic_distribution or {}: - # analytic_ids.add(int(analytic_account)) - # if move_line.full_reconcile_id: - # rec_id = move_line.full_reconcile_id[0] - # if rec_id not in full_reconcile_ids: - # full_reconcile_data.update( - # { - # rec_id: { - # "id": rec_id, - # "name": move_line.full_reconcile_id[1], - # } - # } - # ) - # full_reconcile_ids.add(rec_id) - # acc_id = move_line.account_id[0] - # ml_id = move_line.id - # if acc_id not in gen_ld_data.keys(): - # gen_ld_data[acc_id] = self._initialize_data(foreign_currency) - # gen_ld_data[acc_id]["id"] = acc_id - # gen_ld_data[acc_id]["name"] = move_line.account_id[1] - # if grouped_by: - # gen_ld_data[acc_id][grouped_by] = False - # if acc_id in acc_prt_account_ids: - # item_ids = self._prepare_ml_items(move_line, grouped_by) - # for item in item_ids: - # item_id = item["id"] - # if item_id not in gen_ld_data[acc_id]: - # if grouped_by: - # gen_ld_data[acc_id][grouped_by] = True - # gen_ld_data[acc_id][item_id] = self._initialize_data( - # foreign_currency - # ) - # gen_ld_data[acc_id][item_id]["id"] = item_id - # gen_ld_data[acc_id][item_id]["name"] = item["name"] - # gen_ld_data[acc_id][item_id][ml_id] = self._get_move_line_data( - # move_line - # ) - # gen_ld_data[acc_id][item_id]["fin_bal"][ - # "credit" - # ] += move_line.credit - # gen_ld_data[acc_id][item_id]["fin_bal"]["debit"] += move_line.debit - # gen_ld_data[acc_id][item_id]["fin_bal"][ - # "balance" - # ] += move_line.balance - # if foreign_currency: - # gen_ld_data[acc_id][item_id]["fin_bal"][ - # "bal_curr" - # ] += move_line.amount_currency - # else: - # gen_ld_data[acc_id][ml_id] = self._get_move_line_data(move_line) - # gen_ld_data[acc_id]["fin_bal"]["credit"] += move_line.credit - # gen_ld_data[acc_id]["fin_bal"]["debit"] += move_line.debit - # gen_ld_data[acc_id]["fin_bal"]["balance"] += move_line.balance - # if foreign_currency: - # gen_ld_data[acc_id]["fin_bal"]["bal_curr"] += move_line.amount_currency - def process_ml_data( self, move_lines, @@ -605,42 +532,37 @@ def _get_period_ml_data( analytic_ids = set() full_reconcile_data = {} acc_prt_account_ids = self._get_acc_prt_accounts_ids(company_id, grouped_by) - batch_size = 25000 # Reduced batch size for better memory management + batch_size = 50000 offset = 0 MoveLine = namedtuple("MoveLine", ml_fields) + test_enable = self.env.context.get("test_enable", False) async def fetch_move_lines(offset): - if not self.env.context.get("test_enable", False): - new_cr = sql_db.db_connect(self.env.cr.dbname).cursor() - new_env = api.Environment(new_cr, self.env.uid, self.env.context.copy()) - move_lines = ( - new_env["account.move.line"] - .with_context(prefetch_fields=False) - .search( - domain=domain, - order="date,move_name", - limit=batch_size, - offset=offset, - ) - ) - move_lines_data = move_lines.read(ml_fields) - move_lines = [MoveLine(**line) for line in move_lines_data] - print(move_lines) - return move_lines, new_cr, new_env - else: - move_lines = ( - self.env["account.move.line"] - .with_context(prefetch_fields=False) - .search( - domain=domain, - order="date,move_name", - limit=batch_size, - offset=offset, - ) + new_cr = ( + sql_db.db_connect(self.env.cr.dbname).cursor() + if not test_enable + else None + ) + new_env = ( + api.Environment(new_cr, self.env.uid, self.env.context.copy()) + if not test_enable + else self.env + ) + move_lines = ( + new_env["account.move.line"] + .with_context(prefetch_fields=False) + .search( + domain=domain, + order="date,move_name", + limit=batch_size, + offset=offset, ) - move_lines_data = move_lines.read(ml_fields) - move_lines = [MoveLine(**line) for line in move_lines_data] - return move_lines, None, self.env + ) + move_lines_data = move_lines.with_context(prefetch_fields=False).read( + ml_fields + ) + move_lines = [MoveLine(**line) for line in move_lines_data] + return move_lines, new_cr, new_env async def process_batches(): nonlocal offset @@ -664,12 +586,10 @@ async def process_batches(): acc_prt_account_ids, ) - offset += batch_size - del move_lines - gc.collect() # Invoke garbage collection - if new_cr: new_cr.close() + offset += batch_size + gc.collect() asyncio.run(process_batches()) @@ -772,76 +692,6 @@ def _get_list_grouped_item( list_grouped += [group_item] return account, list_grouped - # def _create_general_ledger( - # self, - # gen_led_data, - # accounts_data, - # grouped_by, - # rec_after_date_to_ids, - # hide_account_at_0, - # ): - # general_ledger = [] - # rounding = self.env.company.currency_id.rounding - # for acc_id in gen_led_data.keys(): - # account = {} - # account.update( - # { - # "code": accounts_data[acc_id]["code"], - # "name": accounts_data[acc_id]["name"], - # "type": "account", - # "currency_id": accounts_data[acc_id]["currency_id"], - # "centralized": accounts_data[acc_id]["centralized"], - # "grouped_by": grouped_by, - # } - # ) - # if grouped_by and not gen_led_data[acc_id][grouped_by]: - # account = self._create_account( - # account, acc_id, gen_led_data, rec_after_date_to_ids - # ) - # if ( - # hide_account_at_0 - # and float_is_zero( - # gen_led_data[acc_id]["init_bal"]["balance"], - # precision_rounding=rounding, - # ) - # and account["move_lines"] == [] - # ): - # continue - # else: - # if grouped_by: - # account, list_grouped = self._get_list_grouped_item( - # gen_led_data[acc_id], - # account, - # rec_after_date_to_ids, - # hide_account_at_0, - # rounding, - # ) - # account.update({"list_grouped": list_grouped}) - # if ( - # hide_account_at_0 - # and float_is_zero( - # gen_led_data[acc_id]["init_bal"]["balance"], - # precision_rounding=rounding, - # ) - # and account["list_grouped"] == [] - # ): - # continue - # else: - # account = self._create_account_not_show_item( - # account, acc_id, gen_led_data, rec_after_date_to_ids, grouped_by - # ) - # if ( - # hide_account_at_0 - # and float_is_zero( - # gen_led_data[acc_id]["init_bal"]["balance"], - # precision_rounding=rounding, - # ) - # and account["move_lines"] == [] - # ): - # continue - # general_ledger += [account] - # return general_ledger - def _create_general_ledger( self, gen_led_data, @@ -1018,7 +868,6 @@ def _get_report_values(self, docids, data): extra_domain, grouped_by, ) - print("Method _get_period_ml_data fully executed ...") general_ledger = self._create_general_ledger( gen_ld_data, accounts_data, @@ -1026,7 +875,6 @@ def _get_report_values(self, docids, data): rec_after_date_to_ids, hide_account_at_0, ) - print("Method _create_general_ledger fully executed ...") if centralize: for account in general_ledger: if account["centralized"]: @@ -1042,9 +890,7 @@ def _get_report_values(self, docids, data): if grouped_by and account[grouped_by]: account[grouped_by] = False del account["list_grouped"] - print("Block centralize passed") general_ledger = sorted(general_ledger, key=lambda k: k["code"]) - print("General ledger sorted") return { "doc_ids": [wizard_id], "doc_model": "general.ledger.report.wizard", diff --git a/account_financial_report/report/general_ledger_xlsx.py b/account_financial_report/report/general_ledger_xlsx.py index e069f2796a8..a169d928758 100644 --- a/account_financial_report/report/general_ledger_xlsx.py +++ b/account_financial_report/report/general_ledger_xlsx.py @@ -5,9 +5,7 @@ # Copyright 2022 Tecnativa - Víctor Martínez # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). -import concurrent - -from odoo import _, models +from odoo import _, api, models, sql_db class GeneralLedgerXslx(models.AbstractModel): @@ -134,7 +132,6 @@ def _get_col_pos_final_balance_label(self): # flake8: noqa: C901 def _generate_report_content(self, workbook, report, data, report_data): - print("Method _generate_report_content") res_data = self.env[ "report.account_financial_report.general_ledger" ]._get_report_values(report, data) @@ -145,382 +142,40 @@ def _generate_report_content(self, workbook, report, data, report_data): analytic_data = res_data["analytic_data"] filter_partner_ids = res_data["filter_partner_ids"] foreign_currency = res_data["foreign_currency"] - - def update_line_with_additional_data(line, account_code): - line.update( - { - "account": account_code, - "journal": journals_data[line["journal_id"]]["code"], - } + test_enable = self.env.context.get("test_enable", False) + # For each account + for account in general_ledger: + new_cr = ( + sql_db.db_connect(self.env.cr.dbname).cursor() + if not test_enable + else None ) - if line["currency_id"]: - line.update( - { - "currency_name": line["currency_id"][1], - "currency_id": line["currency_id"][0], - } - ) - if line["ref_label"] != "Centralized entries": - taxes_description = " ".join( - taxes_data[tax_id]["tax_name"] for tax_id in line["tax_ids"] - ) - if line["tax_line_id"]: - taxes_description += line["tax_line_id"][1] - analytic_distribution = " ".join( - f"{analytic_data[int(account_id)]['name']} {value}%" - if value < 100 - else f"{analytic_data[int(account_id)]['name']}" - for account_id, value in line["analytic_distribution"].items() - ) - line.update( - { - "taxes_description": taxes_description, - "analytic_distribution": analytic_distribution, - } - ) - - def process_account(account): + new_env = ( + api.Environment(new_cr, self.env.uid, self.env.context.copy()) + if not test_enable + else self.env + ) + # Write account title total_bal_curr = account["init_bal"].get("bal_curr", 0) self.write_array_title( - f"{account['code']} - {accounts_data[account['id']]['name']}", + account["code"] + " - " + accounts_data[account["id"]]["name"], report_data, ) - - if "list_grouped" not in account: - self.write_array_header(report_data) - account.update( - { - "initial_debit": account["init_bal"]["debit"], - "initial_credit": account["init_bal"]["credit"], - "initial_balance": account["init_bal"]["balance"], - } - ) - if foreign_currency: - account.update( - {"initial_bal_curr": account["init_bal"]["bal_curr"]} - ) - self.write_initial_balance_from_dict(account, report_data) - - for line in account["move_lines"]: - update_line_with_additional_data(line, account["code"]) - if foreign_currency: - total_bal_curr += line["bal_curr"] - line.update({"total_bal_curr": total_bal_curr}) - self.write_line_from_dict(line, report_data) - - account.update( - { - "final_debit": account["fin_bal"]["debit"], - "final_credit": account["fin_bal"]["credit"], - "final_balance": account["fin_bal"]["balance"], - } - ) - if foreign_currency: - account.update({"final_bal_curr": account["fin_bal"]["bal_curr"]}) - self.write_ending_balance_from_dict(account, report_data) - else: - total_bal_curr = 0 - for group_item in account["list_grouped"]: - self.write_array_title(group_item["name"], report_data) - self.write_array_header(report_data) - account.update( - { - "currency_id": accounts_data[account["id"]]["currency_id"], - "currency_name": accounts_data[account["id"]][ - "currency_name" - ], - } - ) - group_item.update( - { - "initial_debit": group_item["init_bal"]["debit"], - "initial_credit": group_item["init_bal"]["credit"], - "initial_balance": group_item["init_bal"]["balance"], - "type": "partner", - "grouped_by": account.get("grouped_by", ""), - "currency_id": accounts_data[account["id"]]["currency_id"], - "currency_name": accounts_data[account["id"]][ - "currency_name" - ], - } - ) - if foreign_currency: - group_item.update( - {"initial_bal_curr": group_item["init_bal"]["bal_curr"]} - ) - self.write_initial_balance_from_dict(group_item, report_data) - - for line in group_item["move_lines"]: - update_line_with_additional_data(line, account["code"]) - if foreign_currency: - total_bal_curr += line["bal_curr"] - line.update({"total_bal_curr": total_bal_curr}) - self.write_line_from_dict(line, report_data) - - group_item.update( - { - "final_debit": group_item["fin_bal"]["debit"], - "final_credit": group_item["fin_bal"]["credit"], - "final_balance": group_item["fin_bal"]["balance"], - } - ) - if foreign_currency and group_item["currency_id"]: - group_item.update( - {"final_bal_curr": group_item["fin_bal"]["bal_curr"]} - ) - self.write_ending_balance_from_dict(group_item, report_data) - report_data["row_pos"] += 1 - - if not filter_partner_ids: - account.update( - { - "final_debit": account["fin_bal"]["debit"], - "final_credit": account["fin_bal"]["credit"], - "final_balance": account["fin_bal"]["balance"], - } - ) - if foreign_currency and account["currency_id"]: - account.update( - {"final_bal_curr": account["fin_bal"]["bal_curr"]} - ) - self.write_ending_balance_from_dict(account, report_data) - + self.with_env(new_env)._process_account_lines( + account, + report_data, + accounts_data, + journals_data, + taxes_data, + analytic_data, + filter_partner_ids, + foreign_currency, + total_bal_curr, + ) + # 2 lines break report_data["row_pos"] += 2 - - print("Almost fully executed") - # Here is reaching server limit - # for account in general_ledger: - # process_account(account) - - with concurrent.futures.ThreadPoolExecutor() as executor: - futures = [ - executor.submit(process_account, account) for account in general_ledger - ] - for future in concurrent.futures.as_completed(futures): - future.result() - print("Method _generate_report_content fully executed") - - # def _generate_report_content(self, workbook, report, data, report_data): - # res_data = self.env[ - # "report.account_financial_report.general_ledger" - # ]._get_report_values(report, data) - # general_ledger = res_data["general_ledger"] - # accounts_data = res_data["accounts_data"] - # journals_data = res_data["journals_data"] - # taxes_data = res_data["taxes_data"] - # analytic_data = res_data["analytic_data"] - # filter_partner_ids = res_data["filter_partner_ids"] - # foreign_currency = res_data["foreign_currency"] - # # For each account - # for account in general_ledger: - # # Write account title - # total_bal_curr = account["init_bal"].get("bal_curr", 0) - # self.write_array_title( - # account["code"] + " - " + accounts_data[account["id"]]["name"], - # report_data, - # ) - - # if "list_grouped" not in account: - # # Display array header for move lines - # self.write_array_header(report_data) - - # # Display initial balance line for account - # account.update( - # { - # "initial_debit": account["init_bal"]["debit"], - # "initial_credit": account["init_bal"]["credit"], - # "initial_balance": account["init_bal"]["balance"], - # } - # ) - # if foreign_currency: - # account.update( - # {"initial_bal_curr": account["init_bal"]["bal_curr"]} - # ) - # self.write_initial_balance_from_dict(account, report_data) - - # # Display account move lines - # for line in account["move_lines"]: - # line.update( - # { - # "account": account["code"], - # "journal": journals_data[line["journal_id"]]["code"], - # } - # ) - # if line["currency_id"]: - # line.update( - # { - # "currency_name": line["currency_id"][1], - # "currency_id": line["currency_id"][0], - # } - # ) - # if line["ref_label"] != "Centralized entries": - # taxes_description = "" - # analytic_distribution = "" - # for tax_id in line["tax_ids"]: - # taxes_description += taxes_data[tax_id]["tax_name"] + " " - # if line["tax_line_id"]: - # taxes_description += line["tax_line_id"][1] - # for account_id, value in line["analytic_distribution"].items(): - # if value < 100: - # analytic_distribution += "%s %d%% " % ( - # analytic_data[int(account_id)]["name"], - # value, - # ) - # else: - # analytic_distribution += ( - # "%s " % analytic_data[int(account_id)]["name"] - # ) - # line.update( - # { - # "taxes_description": taxes_description, - # "analytic_distribution": analytic_distribution, - # } - # ) - # if foreign_currency: - # total_bal_curr += line["bal_curr"] - # line.update({"total_bal_curr": total_bal_curr}) - # self.write_line_from_dict(line, report_data) - # # Display ending balance line for account - # account.update( - # { - # "final_debit": account["fin_bal"]["debit"], - # "final_credit": account["fin_bal"]["credit"], - # "final_balance": account["fin_bal"]["balance"], - # } - # ) - # if foreign_currency: - # account.update( - # { - # "final_bal_curr": account["fin_bal"]["bal_curr"], - # } - # ) - # self.write_ending_balance_from_dict(account, report_data) - - # else: - # # For each partner - # total_bal_curr = 0 - # for group_item in account["list_grouped"]: - # # Write partner title - # self.write_array_title(group_item["name"], report_data) - - # # Display array header for move lines - # self.write_array_header(report_data) - - # account.update( - # { - # "currency_id": accounts_data[account["id"]]["currency_id"], - # "currency_name": accounts_data[account["id"]][ - # "currency_name" - # ], - # } - # ) - - # # Display initial balance line for partner - # group_item.update( - # { - # "initial_debit": group_item["init_bal"]["debit"], - # "initial_credit": group_item["init_bal"]["credit"], - # "initial_balance": group_item["init_bal"]["balance"], - # "type": "partner", - # "grouped_by": account["grouped_by"] - # if "grouped_by" in account - # else "", - # "currency_id": accounts_data[account["id"]]["currency_id"], - # "currency_name": accounts_data[account["id"]][ - # "currency_name" - # ], - # } - # ) - # if foreign_currency: - # group_item.update( - # { - # "initial_bal_curr": group_item["init_bal"]["bal_curr"], - # } - # ) - # self.write_initial_balance_from_dict(group_item, report_data) - - # # Display account move lines - # for line in group_item["move_lines"]: - # line.update( - # { - # "account": account["code"], - # "journal": journals_data[line["journal_id"]]["code"], - # } - # ) - # if line["currency_id"]: - # line.update( - # { - # "currency_name": line["currency_id"][1], - # "currency_id": line["currency_id"][0], - # } - # ) - # if line["ref_label"] != "Centralized entries": - # taxes_description = "" - # analytic_distribution = "" - # for tax_id in line["tax_ids"]: - # taxes_description += ( - # taxes_data[tax_id]["tax_name"] + " " - # ) - # for account_id, value in line[ - # "analytic_distribution" - # ].items(): - # if value < 100: - # analytic_distribution += "%s %d%% " % ( - # analytic_data[int(account_id)]["name"], - # value, - # ) - # else: - # analytic_distribution += ( - # "%s " % analytic_data[int(account_id)]["name"] - # ) - # line.update( - # { - # "taxes_description": taxes_description, - # "analytic_distribution": analytic_distribution, - # } - # ) - # if foreign_currency: - # total_bal_curr += line["bal_curr"] - # line.update({"total_bal_curr": total_bal_curr}) - # self.write_line_from_dict(line, report_data) - - # # Display ending balance line for partner - # group_item.update( - # { - # "final_debit": group_item["fin_bal"]["debit"], - # "final_credit": group_item["fin_bal"]["credit"], - # "final_balance": group_item["fin_bal"]["balance"], - # } - # ) - # if foreign_currency and group_item["currency_id"]: - # group_item.update( - # { - # "final_bal_curr": group_item["fin_bal"]["bal_curr"], - # } - # ) - # self.write_ending_balance_from_dict(group_item, report_data) - - # # Line break - # report_data["row_pos"] += 1 - - # if not filter_partner_ids: - # account.update( - # { - # "final_debit": account["fin_bal"]["debit"], - # "final_credit": account["fin_bal"]["credit"], - # "final_balance": account["fin_bal"]["balance"], - # } - # ) - # if foreign_currency and account["currency_id"]: - # account.update( - # { - # "final_bal_curr": account["fin_bal"]["bal_curr"], - # } - # ) - # self.write_ending_balance_from_dict(account, report_data) - - # # 2 lines break - # report_data["row_pos"] += 2 + if not test_enable: + new_cr.close() def write_initial_balance_from_dict(self, my_object, report_data): """Specific function to write initial balance for General Ledger""" @@ -548,3 +203,204 @@ def write_ending_balance_from_dict(self, my_object, report_data): return super().write_ending_balance_from_dict( my_object, name, label, report_data ) + + def _process_account_lines( + self, + account, + report_data, + accounts_data, + journals_data, + taxes_data, + analytic_data, + filter_partner_ids, + foreign_currency, + total_bal_curr, + ): + process_method = ( + self._process_single_account + if "list_grouped" not in account + else self._process_grouped_items + ) + process_method( + account, + report_data, + accounts_data, + journals_data, + taxes_data, + analytic_data, + foreign_currency, + total_bal_curr, + filter_partner_ids=filter_partner_ids + if "list_grouped" in account + else False, + ) + + def _process_single_account( + self, + account, + report_data, + accounts_data, + journals_data, + taxes_data, + analytic_data, + foreign_currency, + total_bal_curr, + filter_partner_ids=False, + ): + self.write_array_header(report_data) + account.update( + { + "initial_debit": account["init_bal"]["debit"], + "initial_credit": account["init_bal"]["credit"], + "initial_balance": account["init_bal"]["balance"], + } + ) + if foreign_currency: + account["initial_bal_curr"] = account["init_bal"]["bal_curr"] + self.write_initial_balance_from_dict(account, report_data) + batch_size = 25000 + for i in range(0, len(account["move_lines"]), batch_size): + batch_lines = account["move_lines"][i : i + batch_size] + total_bal_curr = self._process_lines( + batch_lines, + account, + journals_data, + taxes_data, + analytic_data, + foreign_currency, + total_bal_curr, + report_data, + ) + account.update( + { + "final_debit": account["fin_bal"]["debit"], + "final_credit": account["fin_bal"]["credit"], + "final_balance": account["fin_bal"]["balance"], + } + ) + if foreign_currency: + account["final_bal_curr"] = account["fin_bal"]["bal_curr"] + self.write_ending_balance_from_dict(account, report_data) + + def _process_grouped_items( + self, + account, + report_data, + accounts_data, + journals_data, + taxes_data, + analytic_data, + foreign_currency, + total_bal_curr, + filter_partner_ids, + ): + for group_item in account["list_grouped"]: + self.write_array_title(group_item["name"], report_data) + self.write_array_header(report_data) + currency_id = accounts_data[account["id"]]["currency_id"] + currency_name = accounts_data[account["id"]]["currency_name"] + group_item.update( + { + "initial_debit": group_item["init_bal"]["debit"], + "initial_credit": group_item["init_bal"]["credit"], + "initial_balance": group_item["init_bal"]["balance"], + "type": "partner", + "grouped_by": account.get("grouped_by", ""), + "currency_id": currency_id, + "currency_name": currency_name, + } + ) + if foreign_currency: + group_item["initial_bal_curr"] = group_item["init_bal"]["bal_curr"] + self.write_initial_balance_from_dict(group_item, report_data) + batch_size = 25000 # Define the batch size + for i in range(0, len(group_item["move_lines"]), batch_size): + batch_lines = group_item["move_lines"][i : i + batch_size] + total_bal_curr = self._process_lines( + batch_lines, + account, + journals_data, + taxes_data, + analytic_data, + foreign_currency, + total_bal_curr, + report_data, + ) + group_item.update( + { + "final_debit": group_item["fin_bal"]["debit"], + "final_credit": group_item["fin_bal"]["credit"], + "final_balance": group_item["fin_bal"]["balance"], + } + ) + if foreign_currency and group_item["currency_id"]: + group_item["final_bal_curr"] = group_item["fin_bal"]["bal_curr"] + self.write_ending_balance_from_dict(group_item, report_data) + report_data["row_pos"] += 1 + if not filter_partner_ids: + account.update( + { + "final_debit": account["fin_bal"]["debit"], + "final_credit": account["fin_bal"]["credit"], + "final_balance": account["fin_bal"]["balance"], + } + ) + if foreign_currency and account["currency_id"]: + account["final_bal_curr"] = account["fin_bal"]["bal_curr"] + self.write_ending_balance_from_dict(account, report_data) + + def _process_lines( + self, + lines, + account, + journals_data, + taxes_data, + analytic_data, + foreign_currency, + total_bal_curr, + report_data, + ): + for line in lines: + line, total_bal_curr = self._update_line_with_additional_info( + line, + account, + journals_data, + taxes_data, + analytic_data, + foreign_currency, + total_bal_curr, + ) + self.write_line_from_dict(line, report_data) + return total_bal_curr + + def _update_line_with_additional_info( + self, + line, + account, + journals_data, + taxes_data, + analytic_data, + foreign_currency, + total_bal_curr, + ): + line["account"] = account["code"] + line["journal"] = journals_data[line["journal_id"]]["code"] + if line["currency_id"]: + line["currency_name"] = line["currency_id"][1] + line["currency_id"] = line["currency_id"][0] + if line["ref_label"] != "Centralized entries": + line["taxes_description"] = " ".join( + taxes_data[tax_id]["tax_name"] for tax_id in line["tax_ids"] + ) + if line["tax_line_id"]: + line["taxes_description"] += line["tax_line_id"][1] + line["analytic_distribution"] = " ".join( + f"{analytic_data[int(account_id)]['name']} {value}%" + if value < 100 + else f"{analytic_data[int(account_id)]['name']}" + for account_id, value in line["analytic_distribution"].items() + ) + if foreign_currency: + total_bal_curr += line["bal_curr"] + line["total_bal_curr"] = total_bal_curr + return line, total_bal_curr