From ea22e8f431c53660d801b2166872a0fc6f911a19 Mon Sep 17 00:00:00 2001 From: Ninad1306 Date: Fri, 21 Jun 2024 15:29:32 +0530 Subject: [PATCH 01/10] fix: JSON mapping for itc_04 --- .../gst_india/utils/gstr_mapper_utils.py | 161 +++++++++++++++ .../gst_india/utils/itc_04/__init__.py | 42 ++++ .../gst_india/utils/itc_04/itc_04_json_map.py | 187 ++++++++++++++++++ 3 files changed, 390 insertions(+) create mode 100644 india_compliance/gst_india/utils/gstr_mapper_utils.py create mode 100644 india_compliance/gst_india/utils/itc_04/__init__.py create mode 100644 india_compliance/gst_india/utils/itc_04/itc_04_json_map.py diff --git a/india_compliance/gst_india/utils/gstr_mapper_utils.py b/india_compliance/gst_india/utils/gstr_mapper_utils.py new file mode 100644 index 000000000..318240346 --- /dev/null +++ b/india_compliance/gst_india/utils/gstr_mapper_utils.py @@ -0,0 +1,161 @@ +from frappe.utils import flt + + +class GSTRDataMapper: + def convert_to_internal_data_format(self, gov_data): + """ + Converts Gov data format to internal data format for all categories + """ + output = {} + + for category, mapper_class in self.CLASS_MAP.items(): + if not gov_data.get(category): + continue + + output.update( + mapper_class().convert_to_internal_data_format(gov_data.get(category)) + ) + + return output + + def get_category_wise_data( + self, + subcategory_wise_data: dict, + ) -> dict: + """ + returns category wise data from subcategory wise data + + Args: + subcategory_wise_data (dict): subcategory wise data + mapping (dict): subcategory to category mapping + with_subcategory (bool): include subcategory level data + + Returns: + dict: category wise data + + Example (with_subcategory=True): + { + "B2B, SEZ, DE": { + "B2B": data, + ... + } + ... + } + + Example (with_subcategory=False): + { + "B2B, SEZ, DE": data, + ... + } + """ + category_wise_data = {} + for subcategory, category in self.mapping.items(): + if not subcategory_wise_data.get(subcategory.value): + continue + + category_wise_data.setdefault(category.value, []).extend( + subcategory_wise_data.get(subcategory.value, []) + ) + + return category_wise_data + + def convert_to_gov_data_format( + self, internal_data: dict, company_gstin: str + ) -> dict: + """ + converts internal data format to Gov data format for all categories + """ + + output = {} + for category, mapper_class in self.CLASS_MAP.items(): + if not internal_data.get(category): + continue + + output[category] = mapper_class().convert_to_gov_data_format( + internal_data.get(category), company_gstin=company_gstin + ) + + return output + + def summarize_retsum_data( + self, + input_data, + ): + if not input_data: + return [] + + summarized_data = [] + total_values_keys = [ + "total_igst_amount", + "total_cgst_amount", + "total_sgst_amount", + "total_cess_amount", + "total_taxable_value", + ] + amended_data = {key: 0 for key in total_values_keys} + + input_data = {row.get("description"): row for row in input_data} + + def _sum(row): + return flt(sum([row.get(key, 0) for key in total_values_keys]), 2) + + for category, sub_categories in self.category_sub_category_mapping.items(): + category = category.value + if category not in input_data: + continue + + # compute total liability and total amended data + amended_category_data = input_data.get(f"{category} (Amended)", {}) + for key in total_values_keys: + amended_data[key] += amended_category_data.get(key, 0) + + # add category data + if _sum(input_data[category]) == 0: + continue + + summarized_data.append({**input_data.get(category), "indent": 0}) + + # add subcategory data + for sub_category in sub_categories: + sub_category = sub_category.value + if sub_category not in input_data: + continue + + if _sum(input_data[sub_category]) == 0: + continue + + summarized_data.append( + { + **input_data.get(sub_category), + "indent": 1, + "consider_in_total_taxable_value": ( + False + if sub_category + in self.subcategory_not_considered_in_total_taxable_value + else True + ), + "consider_in_total_tax": ( + False + if sub_category + in self.subcategory_not_considered_in_total_tax + else True + ), + } + ) + + # add total amendment liability + if _sum(amended_data) != 0: + summarized_data.extend( + [ + { + "description": "Net Liability from Amendments", + **amended_data, + "indent": 0, + "consider_in_total_taxable_value": True, + "consider_in_total_tax": True, + "no_of_records": 0, + } + ] + ) + + return summarized_data diff --git a/india_compliance/gst_india/utils/itc_04/__init__.py b/india_compliance/gst_india/utils/itc_04/__init__.py new file mode 100644 index 000000000..c344402ad --- /dev/null +++ b/india_compliance/gst_india/utils/itc_04/__init__.py @@ -0,0 +1,42 @@ +from enum import Enum + + +class GovJsonKey(Enum): + """ + Categories / Keys as per Govt JSON file + """ + + TABLE5A = "table5A" + TABLE5B = "table5B" + + +class GovDataField(Enum): + COMPANY_GSTIN = "ctin" + JOB_WORKER_STATE_CODE = "jw_stcd" + ITEMS = "items" + ORIGINAL_CAHLLAN_NUMBER = "o_chnum" + ORIGINAL_CHALLAN_DATE = "o_chdt" + CHALLAN_NUMBER = "jw2_chnum" + CHALLAN_DATE = "jw2_chdt" + NATURE_OF_JOB = "nat_jw" + UOM = "uqc" + QTY = "qty" + DESCRIPTION = "desc" + LOSS_UOM = "lwuqc" + LOSS_QTY = "lwqty" + + +class ITC04_DataField(Enum): + COMPANY_GSTIN = "company_gstin" + JOB_WORKER_STATE_CODE = "jw_state_code" + ITEMS = "items" + ORIGINAL_CAHLLAN_NUMBER = "original_challan_number" + ORIGINAL_CHALLAN_DATE = "original_challan_date" + CHALLAN_NUMBER = "jw_challan_number" + CHALLAN_DATE = "jw_challan_date" + NATURE_OF_JOB = "nature_of_job" + UOM = "uom" + QTY = "qty" + DESCRIPTION = "desc" + LOSS_UOM = "loss_uom" + LOSS_QTY = "loss_qty" diff --git a/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py b/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py new file mode 100644 index 000000000..e50f78922 --- /dev/null +++ b/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py @@ -0,0 +1,187 @@ +from india_compliance.gst_india.constants import STATE_NUMBERS, UOM_MAP +from india_compliance.gst_india.utils.gstr_mapper_utils import GSTRDataMapper +from india_compliance.gst_india.utils.itc_04 import ( + GovDataField, + GovJsonKey, + ITC04_DataField, +) + +############################################################################################################ +### Map Govt JSON to Internal Data Structure ############################################################### +############################################################################################################ + + +class GovDataMapper: + """ + GST Developer API Documentation for Returns - https://developer.gst.gov.in/apiportal/taxpayer/returns + + ITC-04 JSON format - https://developer.gst.gov.in/pages/apiportal/data/Returns/ITC04%20-%20Save/v1.2/ITC04%20-%20Save%20attributes.xlsx + """ + + KEY_MAPPING = {} + + def __init__(self): + self.value_formatters_for_internal = {} + self.value_formatters_for_gov = {} + self.STATE_NUMBERS = self.reverse_dict(STATE_NUMBERS) + + def format_data( + self, data: dict, default_data: dict = None, for_gov: bool = False + ) -> dict: + """ + Objective: Convert Object from one format to another. + eg: Govt JSON to Internal Data Structure + + Args: + data (dict): Data to be converted + default_data (dict, optional): Default Data to be added. Hardcoded values. + for_gov (bool, optional): If the data is to be converted to Govt JSON. Defaults to False. + else it will be converted to Internal Data Structure. + + Steps: + 1. Use key mapping to map the keys from one format to another. + 2. Use value formatters to format the values of the keys. + 3. Round values + """ + output = {} + + if default_data: + output.update(default_data) + + key_mapping = self.KEY_MAPPING.copy() + + if for_gov: + key_mapping = self.reverse_dict(key_mapping) + + value_formatters = ( + self.value_formatters_for_gov + if for_gov + else self.value_formatters_for_internal + ) + + for old_key, new_key in key_mapping.items(): + invoice_data_value = data.get(old_key, "") + + if not for_gov and old_key == "flag": + continue + + if not (invoice_data_value or invoice_data_value == 0): + # continue if value is None or empty object + continue + + value_formatter = value_formatters.get(old_key) + + if callable(value_formatter): + output[new_key] = value_formatter(invoice_data_value) + else: + output[new_key] = invoice_data_value + + return output + + def map_uom(self, uom): + uom = uom.upper() + + if "-" in uom: + return uom.split("-")[0] + + if uom in UOM_MAP: + return f"{uom}-{UOM_MAP[uom]}" + + return f"OTH-{UOM_MAP.get('OTH')}" + + def map_place_of_supply(self, state_code): + if state_code.isnumeric(): + return f"{state_code}-{self.STATE_NUMBERS.get(state_code)}" + + return state_code.split("-")[0] + + def reverse_dict(self, data): + return {v: k for k, v in data.items()} + + +class TABLE5A(GovDataMapper): + CATEGORY = GovJsonKey.TABLE5A.value + + KEY_MAPPING = { + GovDataField.COMPANY_GSTIN.value: ITC04_DataField.COMPANY_GSTIN.value, + GovDataField.JOB_WORKER_STATE_CODE.value: ITC04_DataField.JOB_WORKER_STATE_CODE.value, + GovDataField.ITEMS.value: ITC04_DataField.ITEMS.value, + GovDataField.ORIGINAL_CAHLLAN_NUMBER.value: ITC04_DataField.ORIGINAL_CAHLLAN_NUMBER.value, + GovDataField.ORIGINAL_CHALLAN_DATE.value: ITC04_DataField.ORIGINAL_CHALLAN_DATE.value, + GovDataField.CHALLAN_NUMBER.value: ITC04_DataField.CHALLAN_NUMBER.value, + GovDataField.CHALLAN_DATE.value: ITC04_DataField.CHALLAN_DATE.value, + GovDataField.NATURE_OF_JOB.value: ITC04_DataField.NATURE_OF_JOB.value, + GovDataField.UOM.value: ITC04_DataField.UOM.value, + GovDataField.QTY.value: ITC04_DataField.QTY.value, + GovDataField.DESCRIPTION.value: ITC04_DataField.DESCRIPTION.value, + GovDataField.LOSS_UOM.value: ITC04_DataField.LOSS_UOM.value, + GovDataField.LOSS_QTY.value: ITC04_DataField.LOSS_QTY.value, + } + + def __init__(self): + super().__init__() + + self.value_formatters_for_internal = { + GovDataField.ITEMS.value: self.format_item_for_internal, + GovDataField.UOM.value: self.map_uom, + GovDataField.LOSS_UOM.value: self.map_uom, + GovDataField.JOB_WORKER_STATE_CODE.value: self.map_place_of_supply, + } + + self.value_formatters_for_gov = { + ITC04_DataField.ITEMS.value: self.format_item_for_gov, + ITC04_DataField.UOM.value: self.map_uom, + ITC04_DataField.LOSS_UOM.value: self.map_uom, + ITC04_DataField.JOB_WORKER_STATE_CODE.value: self.map_place_of_supply, + } + + def convert_to_internal_data_format(self, input_data): + output = {} + + for invoice in input_data: + original_challan_number = invoice.get(GovDataField.ITEMS.value)[0].get( + GovDataField.ORIGINAL_CAHLLAN_NUMBER.value + ) + output[original_challan_number] = self.format_data(invoice) + + return {self.CATEGORY: output} + + def convert_to_gov_data_format(self, input_data, **kwargs): + output = [] + + for invoice in input_data: + formatted_data = self.format_data(invoice, for_gov=True) + + output.append(formatted_data) + + return output + + def format_item_for_internal(self, items): + return [ + { + **self.format_data(item), + } + for item in items + ] + + def format_item_for_gov(self, items, *args): + return [ + { + **self.format_data(item, for_gov=True), + } + for item in items + ] + + +class TABLE5B(TABLE5A): + CATEGORY = GovJsonKey.TABLE5B.value + + def __init__(self): + super().__init__() + + +class ITC04DataMapper(GSTRDataMapper): + CLASS_MAP = { + GovJsonKey.TABLE5A.value: TABLE5A, + GovJsonKey.TABLE5B.value: TABLE5B, + } From fddcc721a5d0b7b8ff41176e8f54c4c02a5d7d23 Mon Sep 17 00:00:00 2001 From: Ninad1306 Date: Fri, 21 Jun 2024 16:13:45 +0530 Subject: [PATCH 02/10] refactor: generalize JSON mapping for broader use --- .../doctype/gstr_1_beta/gstr_1_export.py | 11 +- .../doctype/gstr_1_log/gstr_1_log.py | 4 +- .../gst_india/utils/gstr_1/gstr_1_download.py | 8 +- .../gst_india/utils/gstr_1/gstr_1_json_map.py | 207 +++--------------- .../utils/gstr_1/test_gstr_1_json_map.py | 9 +- 5 files changed, 46 insertions(+), 193 deletions(-) diff --git a/india_compliance/gst_india/doctype/gstr_1_beta/gstr_1_export.py b/india_compliance/gst_india/doctype/gstr_1_beta/gstr_1_export.py index e0f098f30..2bb1fe955 100644 --- a/india_compliance/gst_india/doctype/gstr_1_beta/gstr_1_export.py +++ b/india_compliance/gst_india/doctype/gstr_1_beta/gstr_1_export.py @@ -21,10 +21,7 @@ GSTR1_ItemField, GSTR1_SubCategory, ) -from india_compliance.gst_india.utils.gstr_1.gstr_1_json_map import ( - convert_to_gov_data_format, - get_category_wise_data, -) +from india_compliance.gst_india.utils.gstr_1.gstr_1_json_map import GSTR1DataMapper class ExcelWidth(Enum): @@ -58,7 +55,7 @@ def process_data(self, input_data): 2. Format/Transform the data to match the Gov Excel format """ - category_wise_data = get_category_wise_data(input_data) + category_wise_data = GSTR1DataMapper().get_category_wise_data(input_data) processed_data = {} for category, data in category_wise_data.items(): @@ -1141,7 +1138,7 @@ def __init__(self, company_gstin, month_or_quarter, year): self.summary = gstr1_log.load_data("reconcile_summary")["reconcile_summary"] data = gstr1_log.load_data("reconcile")["reconcile"] - self.data = get_category_wise_data(data) + self.data = GSTR1DataMapper().get_category_wise_data(data) def export_data(self): excel = ExcelExporter() @@ -2080,7 +2077,7 @@ def download_gstr_1_json( "data": { "gstin": company_gstin, "fp": period, - **convert_to_gov_data_format(data, company_gstin), + **GSTR1DataMapper().convert_to_gov_data_format(data, company_gstin), }, "filename": f"GSTR-1-Gov-{company_gstin}-{period}.json", } diff --git a/india_compliance/gst_india/doctype/gstr_1_log/gstr_1_log.py b/india_compliance/gst_india/doctype/gstr_1_log/gstr_1_log.py index 8bd15c7a6..6b3279da9 100644 --- a/india_compliance/gst_india/doctype/gstr_1_log/gstr_1_log.py +++ b/india_compliance/gst_india/doctype/gstr_1_log/gstr_1_log.py @@ -22,7 +22,7 @@ ) from india_compliance.gst_india.utils.gstr_1.gstr_1_json_map import ( GSTR1BooksData, - summarize_retsum_data, + GSTR1DataMapper, ) from india_compliance.gst_india.utils.gstr_utils import request_otp @@ -41,7 +41,7 @@ def get_summarized_data(self, data, is_filed=False): Helper function to summarize data for each sub-category """ if is_filed and data.get("summary"): - return summarize_retsum_data(data.get("summary")) + return GSTR1DataMapper().summarize_retsum_data(data.get("summary")) subcategory_summary = self.get_subcategory_summary(data) diff --git a/india_compliance/gst_india/utils/gstr_1/gstr_1_download.py b/india_compliance/gst_india/utils/gstr_1/gstr_1_download.py index 94a4d2102..5d27d1b39 100644 --- a/india_compliance/gst_india/utils/gstr_1/gstr_1_download.py +++ b/india_compliance/gst_india/utils/gstr_1/gstr_1_download.py @@ -6,9 +6,7 @@ from india_compliance.gst_india.doctype.gstr_import_log.gstr_import_log import ( create_import_log, ) -from india_compliance.gst_india.utils.gstr_1.gstr_1_json_map import ( - convert_to_internal_data_format, -) +from india_compliance.gst_india.utils.gstr_1.gstr_1_json_map import GSTR1DataMapper UNFILED_ACTIONS = [ "B2B", @@ -77,7 +75,7 @@ def download_gstr1_json_data(gstr1_log): json_data.update(response) - mapped_data = convert_to_internal_data_format(json_data) + mapped_data = GSTR1DataMapper().convert_to_internal_data_format(json_data) gstr1_log.update_json_for(data_field, mapped_data, reset_reconcile=True) if is_queued: @@ -109,7 +107,7 @@ def save_gstr_1(gstin, return_period, json_data, return_type): title=_("Invalid Response Received."), ) - mapped_data = convert_to_internal_data_format(json_data) + mapped_data = GSTR1DataMapper().convert_to_internal_data_format(json_data) gstr1_log = frappe.get_doc("GSTR-1 Log", f"{return_period}-{gstin}") gstr1_log.update_json_for(data_field, mapped_data, overwrite=False) diff --git a/india_compliance/gst_india/utils/gstr_1/gstr_1_json_map.py b/india_compliance/gst_india/utils/gstr_1/gstr_1_json_map.py index d57409262..fb1ae0282 100644 --- a/india_compliance/gst_india/utils/gstr_1/gstr_1_json_map.py +++ b/india_compliance/gst_india/utils/gstr_1/gstr_1_json_map.py @@ -8,8 +8,10 @@ GSTR1DocumentIssuedSummary, GSTR11A11BData, ) -from india_compliance.gst_india.utils import get_gst_accounts_by_type -from india_compliance.gst_india.utils.__init__ import get_party_for_gstin +from india_compliance.gst_india.utils import ( + get_gst_accounts_by_type, + get_party_for_gstin, +) from india_compliance.gst_india.utils.gstr_1 import ( CATEGORY_SUB_CATEGORY_MAPPING, SUB_CATEGORY_GOV_CATEGORY_MAPPING, @@ -24,6 +26,7 @@ GSTR1_SubCategory, ) from india_compliance.gst_india.utils.gstr_1.gstr_1_data import GSTR1Invoices +from india_compliance.gst_india.utils.gstr_mapper_utils import GSTRDataMapper ############################################################################################################ ### Map Govt JSON to Internal Data Structure ############################################################### @@ -1834,179 +1837,35 @@ def map_document_types(self, doc_type, *args): return self.SECTION_NAMES.get(doc_type, doc_type) -CLASS_MAP = { - GovJsonKey.B2B.value: B2B, - GovJsonKey.B2CL.value: B2CL, - GovJsonKey.EXP.value: Exports, - GovJsonKey.B2CS.value: B2CS, - GovJsonKey.NIL_EXEMPT.value: NilRated, - GovJsonKey.CDNR.value: CDNR, - GovJsonKey.CDNUR.value: CDNUR, - GovJsonKey.HSN.value: HSNSUM, - GovJsonKey.DOC_ISSUE.value: DOC_ISSUE, - GovJsonKey.AT.value: AT, - GovJsonKey.TXP.value: TXPD, - GovJsonKey.SUPECOM.value: SUPECOM, - GovJsonKey.RET_SUM.value: RETSUM, -} - - -def convert_to_internal_data_format(gov_data): - """ - Converts Gov data format to internal data format for all categories - """ - output = {} - - for category, mapper_class in CLASS_MAP.items(): - if not gov_data.get(category): - continue - - output.update( - mapper_class().convert_to_internal_data_format(gov_data.get(category)) - ) - - return output - - -def get_category_wise_data( - subcategory_wise_data: dict, - mapping: dict = SUB_CATEGORY_GOV_CATEGORY_MAPPING, -) -> dict: - """ - returns category wise data from subcategory wise data - - Args: - subcategory_wise_data (dict): subcategory wise data - mapping (dict): subcategory to category mapping - with_subcategory (bool): include subcategory level data - - Returns: - dict: category wise data - - Example (with_subcategory=True): - { - "B2B, SEZ, DE": { - "B2B": data, - ... - } - ... - } - - Example (with_subcategory=False): - { - "B2B, SEZ, DE": data, - ... - } - """ - category_wise_data = {} - for subcategory, category in mapping.items(): - if not subcategory_wise_data.get(subcategory.value): - continue - - category_wise_data.setdefault(category.value, []).extend( - subcategory_wise_data.get(subcategory.value, []) - ) - - return category_wise_data - - -def convert_to_gov_data_format(internal_data: dict, company_gstin: str) -> dict: - """ - converts internal data format to Gov data format for all categories - """ - - category_wise_data = get_category_wise_data(internal_data) - - output = {} - for category, mapper_class in CLASS_MAP.items(): - if not category_wise_data.get(category): - continue - - output[category] = mapper_class().convert_to_gov_data_format( - category_wise_data.get(category), company_gstin=company_gstin - ) - - return output - - -def summarize_retsum_data(input_data): - if not input_data: - return [] - - summarized_data = [] - total_values_keys = [ - "total_igst_amount", - "total_cgst_amount", - "total_sgst_amount", - "total_cess_amount", - "total_taxable_value", - ] - amended_data = {key: 0 for key in total_values_keys} - - input_data = {row.get("description"): row for row in input_data} - - def _sum(row): - return flt(sum([row.get(key, 0) for key in total_values_keys]), 2) - - for category, sub_categories in CATEGORY_SUB_CATEGORY_MAPPING.items(): - category = category.value - if category not in input_data: - continue - - # compute total liability and total amended data - amended_category_data = input_data.get(f"{category} (Amended)", {}) - for key in total_values_keys: - amended_data[key] += amended_category_data.get(key, 0) - - # add category data - if _sum(input_data[category]) == 0: - continue - - summarized_data.append({**input_data.get(category), "indent": 0}) - - # add subcategory data - for sub_category in sub_categories: - sub_category = sub_category.value - if sub_category not in input_data: - continue - - if _sum(input_data[sub_category]) == 0: - continue - - summarized_data.append( - { - **input_data.get(sub_category), - "indent": 1, - "consider_in_total_taxable_value": ( - False - if sub_category - in SUBCATEGORIES_NOT_CONSIDERED_IN_TOTAL_TAXABLE_VALUE - else True - ), - "consider_in_total_tax": ( - False - if sub_category in SUBCATEGORIES_NOT_CONSIDERED_IN_TOTAL_TAX - else True - ), - } - ) - - # add total amendment liability - if _sum(amended_data) != 0: - summarized_data.extend( - [ - { - "description": "Net Liability from Amendments", - **amended_data, - "indent": 0, - "consider_in_total_taxable_value": True, - "consider_in_total_tax": True, - "no_of_records": 0, - } - ] - ) +class GSTR1DataMapper(GSTRDataMapper): + CLASS_MAP = { + GovJsonKey.B2B.value: B2B, + GovJsonKey.B2CL.value: B2CL, + GovJsonKey.EXP.value: Exports, + GovJsonKey.B2CS.value: B2CS, + GovJsonKey.NIL_EXEMPT.value: NilRated, + GovJsonKey.CDNR.value: CDNR, + GovJsonKey.CDNUR.value: CDNUR, + GovJsonKey.HSN.value: HSNSUM, + GovJsonKey.DOC_ISSUE.value: DOC_ISSUE, + GovJsonKey.AT.value: AT, + GovJsonKey.TXP.value: TXPD, + GovJsonKey.SUPECOM.value: SUPECOM, + GovJsonKey.RET_SUM.value: RETSUM, + } - return summarized_data + category_sub_category_mapping = CATEGORY_SUB_CATEGORY_MAPPING + subcategories_not_considered_in_total_tax = ( + SUBCATEGORIES_NOT_CONSIDERED_IN_TOTAL_TAX + ) + subcategories_not_considered_in_total_taxable_value = ( + SUBCATEGORIES_NOT_CONSIDERED_IN_TOTAL_TAXABLE_VALUE + ) + mapping = SUB_CATEGORY_GOV_CATEGORY_MAPPING + + def convert_to_gov_data_format(self, internal_data, company_gstin): + category_wise_data = self.get_category_wise_data(internal_data) + return super().convert_to_gov_data_format(category_wise_data, company_gstin) #################################################################################################### diff --git a/india_compliance/gst_india/utils/gstr_1/test_gstr_1_json_map.py b/india_compliance/gst_india/utils/gstr_1/test_gstr_1_json_map.py index a602bcbb0..c5f1d9ec1 100644 --- a/india_compliance/gst_india/utils/gstr_1/test_gstr_1_json_map.py +++ b/india_compliance/gst_india/utils/gstr_1/test_gstr_1_json_map.py @@ -5,7 +5,6 @@ from india_compliance.gst_india.doctype.gstr_1_log.gstr_1_log import GenerateGSTR1 from india_compliance.gst_india.utils import get_party_for_gstin as _get_party_for_gstin from india_compliance.gst_india.utils.gstr_1 import ( - SUB_CATEGORY_GOV_CATEGORY_MAPPING, GovDataField, GSTR1_B2B_InvoiceType, GSTR1_DataField, @@ -24,8 +23,8 @@ SUPECOM, TXPD, Exports, + GSTR1DataMapper, NilRated, - get_category_wise_data, ) @@ -39,9 +38,9 @@ def normalize_data(data): def process_mapped_data(data): return list( - get_category_wise_data( - normalize_data(copy.deepcopy(data)), SUB_CATEGORY_GOV_CATEGORY_MAPPING - ).values() + GSTR1DataMapper() + .get_category_wise_data(normalize_data(copy.deepcopy(data))) + .values() )[0] From 06b2e654782e3c0d8a65211ad9384ab6a8cb50a2 Mon Sep 17 00:00:00 2001 From: Ninad1306 Date: Mon, 24 Jun 2024 17:39:20 +0530 Subject: [PATCH 03/10] fix: JSON mapping changes --- .../gst_india/utils/gstr_1/gstr_1_json_map.py | 329 ++++++++++-------- .../gst_india/utils/gstr_mapper_utils.py | 221 +++++------- .../gst_india/utils/itc_04/__init__.py | 47 ++- .../gst_india/utils/itc_04/itc_04_json_map.py | 238 ++++++++----- 4 files changed, 463 insertions(+), 372 deletions(-) diff --git a/india_compliance/gst_india/utils/gstr_1/gstr_1_json_map.py b/india_compliance/gst_india/utils/gstr_1/gstr_1_json_map.py index fb1ae0282..597eaf486 100644 --- a/india_compliance/gst_india/utils/gstr_1/gstr_1_json_map.py +++ b/india_compliance/gst_india/utils/gstr_1/gstr_1_json_map.py @@ -3,7 +3,7 @@ import frappe from frappe.utils import flt -from india_compliance.gst_india.constants import STATE_NUMBERS, UOM_MAP +from india_compliance.gst_india.constants import UOM_MAP from india_compliance.gst_india.report.gstr_1.gstr_1 import ( GSTR1DocumentIssuedSummary, GSTR11A11BData, @@ -26,21 +26,20 @@ GSTR1_SubCategory, ) from india_compliance.gst_india.utils.gstr_1.gstr_1_data import GSTR1Invoices -from india_compliance.gst_india.utils.gstr_mapper_utils import GSTRDataMapper +from india_compliance.gst_india.utils.gstr_mapper_utils import GovDataMapper ############################################################################################################ ### Map Govt JSON to Internal Data Structure ############################################################### ############################################################################################################ -class GovDataMapper: +class GSTR1DataMapper(GovDataMapper): """ GST Developer API Documentation for Returns - https://developer.gst.gov.in/apiportal/taxpayer/returns GSTR-1 JSON format - https://developer.gst.gov.in/pages/apiportal/data/Returns/GSTR1%20-%20Save%20GSTR1%20data/v4.0/GSTR1%20-%20Save%20GSTR1%20data%20attributes.xlsx """ - KEY_MAPPING = {} # default item amounts DEFAULT_ITEM_AMOUNTS = { GSTR1_ItemField.TAXABLE_VALUE.value: 0, @@ -71,106 +70,10 @@ class GovDataMapper: } def __init__(self): - self.set_total_defaults() - - self.value_formatters_for_internal = {} - self.value_formatters_for_gov = {} + super().__init__() self.gstin_party_map = {} - # value formatting constants - - self.STATE_NUMBERS = self.reverse_dict(STATE_NUMBERS) - - def format_data( - self, data: dict, default_data: dict = None, for_gov: bool = False - ) -> dict: - """ - Objective: Convert Object from one format to another. - eg: Govt JSON to Internal Data Structure - - Args: - data (dict): Data to be converted - default_data (dict, optional): Default Data to be added. Hardcoded values. - for_gov (bool, optional): If the data is to be converted to Govt JSON. Defaults to False. - else it will be converted to Internal Data Structure. - - Steps: - 1. Use key mapping to map the keys from one format to another. - 2. Use value formatters to format the values of the keys. - 3. Round values - """ - output = {} - - if default_data: - output.update(default_data) - - key_mapping = self.KEY_MAPPING.copy() - - if for_gov: - key_mapping = self.reverse_dict(key_mapping) - - value_formatters = ( - self.value_formatters_for_gov - if for_gov - else self.value_formatters_for_internal - ) - - for old_key, new_key in key_mapping.items(): - invoice_data_value = data.get(old_key, "") - - if not for_gov and old_key == "flag": - continue - - if new_key in self.DISCARD_IF_ZERO_FIELDS and not invoice_data_value: - continue - - if not (invoice_data_value or invoice_data_value == 0): - # continue if value is None or empty object - continue - - value_formatter = value_formatters.get(old_key) - - if callable(value_formatter): - output[new_key] = value_formatter(invoice_data_value, data) - else: - output[new_key] = invoice_data_value - - if new_key in self.FLOAT_FIELDS: - output[new_key] = flt(output[new_key], 2) - - return output - - # common utils - - def update_totals(self, invoice, items): - """ - Update item totals to the invoice row - """ - total_data = self.TOTAL_DEFAULTS.copy() - - for item in items: - for field, value in item.items(): - total_field = f"total_{field}" - - if total_field not in total_data: - continue - - invoice[total_field] = invoice.setdefault(total_field, 0) + value - - def set_total_defaults(self): - self.TOTAL_DEFAULTS = { - f"total_{key}": 0 for key in self.DEFAULT_ITEM_AMOUNTS.keys() - } - - def reverse_dict(self, data): - return {v: k for k, v in data.items()} # common value formatters - def map_place_of_supply(self, pos, *args): - if pos.isnumeric(): - return f"{pos}-{self.STATE_NUMBERS.get(pos)}" - - return pos.split("-")[0] - def format_item_for_internal(self, items, *args): return [ { @@ -204,7 +107,7 @@ def format_date_for_gov(self, date, *args): return datetime.strptime(date, "%Y-%m-%d").strftime("%d-%m-%Y") -class B2B(GovDataMapper): +class B2B(GSTR1DataMapper): """ GST API Version - v4.0 @@ -379,7 +282,7 @@ def document_category_mapping(self, sub_category, data): return self.DOCUMENT_CATEGORIES.get(sub_category, sub_category) -class B2CL(GovDataMapper): +class B2CL(GSTR1DataMapper): """ GST API Version - v4.0 @@ -505,7 +408,7 @@ def convert_to_gov_data_format(self, input_data, **kwargs): return list(pos_data.values()) -class Exports(GovDataMapper): +class Exports(GSTR1DataMapper): """ GST API Version - v4.0 @@ -651,7 +554,7 @@ def format_item_for_gov(self, items, *args): return [self.format_data(item, for_gov=True) for item in items] -class B2CS(GovDataMapper): +class B2CS(GSTR1DataMapper): """ GST API Version - v4.0 @@ -747,7 +650,7 @@ def format_data(self, data, default_data=None, for_gov=False): return data -class NilRated(GovDataMapper): +class NilRated(GSTR1DataMapper): """ GST API Version - v4.0 @@ -852,7 +755,7 @@ def document_category_mapping(self, doc_category, data): return self.DOCUMENT_CATEGORIES.get(doc_category, doc_category) -class CDNR(GovDataMapper): +class CDNR(GSTR1DataMapper): """ GST API Version - v4.0 @@ -1040,7 +943,7 @@ def format_doc_value(self, value, data): return value * -1 if data[GovDataField.NOTE_TYPE.value] == "C" else value -class CDNUR(GovDataMapper): +class CDNUR(GSTR1DataMapper): """ GST API Version - v4.0 @@ -1178,7 +1081,7 @@ def format_doc_value(self, value, data): return value * -1 if data[GovDataField.NOTE_TYPE.value] == "C" else value -class HSNSUM(GovDataMapper): +class HSNSUM(GSTR1DataMapper): """ GST API Version - v4.0 @@ -1298,7 +1201,7 @@ def map_uom(self, uom, data=None): return f"OTH-{UOM_MAP.get('OTH')}" -class AT(GovDataMapper): +class AT(GSTR1DataMapper): """ GST API Version - v4.0 @@ -1479,7 +1382,7 @@ class TXPD(AT): MULTIPLIER = -1 -class DOC_ISSUE(GovDataMapper): +class DOC_ISSUE(GSTR1DataMapper): """ GST API Version - v4.0 @@ -1615,7 +1518,7 @@ def get_document_nature(self, doc_nature, *args): return self.DOCUMENT_NATURE.get(doc_nature, doc_nature) -class SUPECOM(GovDataMapper): +class SUPECOM(GSTR1DataMapper): """ GST API Version - v4.0 @@ -1692,7 +1595,7 @@ def convert_to_gov_data_format(self, input_data, **kwargs): return output -class RETSUM(GovDataMapper): +class RETSUM(GSTR1DataMapper): """ Convert GSTR-1 Summary as returned by the API to the internal format @@ -1837,35 +1740,179 @@ def map_document_types(self, doc_type, *args): return self.SECTION_NAMES.get(doc_type, doc_type) -class GSTR1DataMapper(GSTRDataMapper): - CLASS_MAP = { - GovJsonKey.B2B.value: B2B, - GovJsonKey.B2CL.value: B2CL, - GovJsonKey.EXP.value: Exports, - GovJsonKey.B2CS.value: B2CS, - GovJsonKey.NIL_EXEMPT.value: NilRated, - GovJsonKey.CDNR.value: CDNR, - GovJsonKey.CDNUR.value: CDNUR, - GovJsonKey.HSN.value: HSNSUM, - GovJsonKey.DOC_ISSUE.value: DOC_ISSUE, - GovJsonKey.AT.value: AT, - GovJsonKey.TXP.value: TXPD, - GovJsonKey.SUPECOM.value: SUPECOM, - GovJsonKey.RET_SUM.value: RETSUM, - } +CLASS_MAP = { + GovJsonKey.B2B.value: B2B, + GovJsonKey.B2CL.value: B2CL, + GovJsonKey.EXP.value: Exports, + GovJsonKey.B2CS.value: B2CS, + GovJsonKey.NIL_EXEMPT.value: NilRated, + GovJsonKey.CDNR.value: CDNR, + GovJsonKey.CDNUR.value: CDNUR, + GovJsonKey.HSN.value: HSNSUM, + GovJsonKey.DOC_ISSUE.value: DOC_ISSUE, + GovJsonKey.AT.value: AT, + GovJsonKey.TXP.value: TXPD, + GovJsonKey.SUPECOM.value: SUPECOM, + GovJsonKey.RET_SUM.value: RETSUM, +} + + +def convert_to_internal_data_format(gov_data): + """ + Converts Gov data format to internal data format for all categories + """ + output = {} + + for category, mapper_class in CLASS_MAP.items(): + if not gov_data.get(category): + continue + + output.update( + mapper_class().convert_to_internal_data_format(gov_data.get(category)) + ) + + return output + + +def get_category_wise_data( + subcategory_wise_data: dict, + mapping: dict = SUB_CATEGORY_GOV_CATEGORY_MAPPING, +) -> dict: + """ + returns category wise data from subcategory wise data + + Args: + subcategory_wise_data (dict): subcategory wise data + mapping (dict): subcategory to category mapping + with_subcategory (bool): include subcategory level data + + Returns: + dict: category wise data + + Example (with_subcategory=True): + { + "B2B, SEZ, DE": { + "B2B": data, + ... + } + ... + } + + Example (with_subcategory=False): + { + "B2B, SEZ, DE": data, + ... + } + """ + category_wise_data = {} + for subcategory, category in mapping.items(): + if not subcategory_wise_data.get(subcategory.value): + continue + + category_wise_data.setdefault(category.value, []).extend( + subcategory_wise_data.get(subcategory.value, []) + ) + + return category_wise_data + + +def convert_to_gov_data_format(internal_data: dict, company_gstin: str) -> dict: + """ + converts internal data format to Gov data format for all categories + """ + + category_wise_data = get_category_wise_data(internal_data) + + output = {} + for category, mapper_class in CLASS_MAP.items(): + if not category_wise_data.get(category): + continue + + output[category] = mapper_class().convert_to_gov_data_format( + category_wise_data.get(category), company_gstin=company_gstin + ) + + return output + + +def summarize_retsum_data(input_data): + if not input_data: + return [] + + summarized_data = [] + total_values_keys = [ + "total_igst_amount", + "total_cgst_amount", + "total_sgst_amount", + "total_cess_amount", + "total_taxable_value", + ] + amended_data = {key: 0 for key in total_values_keys} + + input_data = {row.get("description"): row for row in input_data} + + def _sum(row): + return flt(sum([row.get(key, 0) for key in total_values_keys]), 2) + + for category, sub_categories in CATEGORY_SUB_CATEGORY_MAPPING.items(): + category = category.value + if category not in input_data: + continue + + # compute total liability and total amended data + amended_category_data = input_data.get(f"{category} (Amended)", {}) + for key in total_values_keys: + amended_data[key] += amended_category_data.get(key, 0) + + # add category data + if _sum(input_data[category]) == 0: + continue + + summarized_data.append({**input_data.get(category), "indent": 0}) + + # add subcategory data + for sub_category in sub_categories: + sub_category = sub_category.value + if sub_category not in input_data: + continue + + if _sum(input_data[sub_category]) == 0: + continue + + summarized_data.append( + { + **input_data.get(sub_category), + "indent": 1, + "consider_in_total_taxable_value": ( + False + if sub_category + in SUBCATEGORIES_NOT_CONSIDERED_IN_TOTAL_TAXABLE_VALUE + else True + ), + "consider_in_total_tax": ( + False + if sub_category in SUBCATEGORIES_NOT_CONSIDERED_IN_TOTAL_TAX + else True + ), + } + ) + + # add total amendment liability + if _sum(amended_data) != 0: + summarized_data.extend( + [ + { + "description": "Net Liability from Amendments", + **amended_data, + "indent": 0, + "consider_in_total_taxable_value": True, + "consider_in_total_tax": True, + "no_of_records": 0, + } + ] + ) - category_sub_category_mapping = CATEGORY_SUB_CATEGORY_MAPPING - subcategories_not_considered_in_total_tax = ( - SUBCATEGORIES_NOT_CONSIDERED_IN_TOTAL_TAX - ) - subcategories_not_considered_in_total_taxable_value = ( - SUBCATEGORIES_NOT_CONSIDERED_IN_TOTAL_TAXABLE_VALUE - ) - mapping = SUB_CATEGORY_GOV_CATEGORY_MAPPING - - def convert_to_gov_data_format(self, internal_data, company_gstin): - category_wise_data = self.get_category_wise_data(internal_data) - return super().convert_to_gov_data_format(category_wise_data, company_gstin) + return summarized_data #################################################################################################### diff --git a/india_compliance/gst_india/utils/gstr_mapper_utils.py b/india_compliance/gst_india/utils/gstr_mapper_utils.py index 318240346..48171c454 100644 --- a/india_compliance/gst_india/utils/gstr_mapper_utils.py +++ b/india_compliance/gst_india/utils/gstr_mapper_utils.py @@ -1,161 +1,116 @@ from frappe.utils import flt +from india_compliance.gst_india.constants import STATE_NUMBERS -class GSTRDataMapper: - def convert_to_internal_data_format(self, gov_data): - """ - Converts Gov data format to internal data format for all categories - """ - output = {} - for category, mapper_class in self.CLASS_MAP.items(): - if not gov_data.get(category): - continue +class GovDataMapper: + """ + GST Developer API Documentation for Returns - https://developer.gst.gov.in/apiportal/taxpayer/returns - output.update( - mapper_class().convert_to_internal_data_format(gov_data.get(category)) - ) + GSTR-1 JSON format - https://developer.gst.gov.in/pages/apiportal/data/Returns/GSTR1%20-%20Save%20GSTR1%20data/v4.0/GSTR1%20-%20Save%20GSTR1%20data%20attributes.xlsx + """ - return output + KEY_MAPPING = {} + FLOAT_FIELDS = {} + DISCARD_IF_ZERO_FIELDS = {} + TOTAL_DEFAULTS = {} + DEFAULT_ITEM_AMOUNTS = {} + + def __init__(self): + # value formatting constants + self.value_formatters_for_internal = {} + self.value_formatters_for_gov = {} + + self.STATE_NUMBERS = self.reverse_dict(STATE_NUMBERS) + self.set_total_defaults() - def get_category_wise_data( - self, - subcategory_wise_data: dict, + def format_data( + self, data: dict, default_data: dict = None, for_gov: bool = False ) -> dict: """ - returns category wise data from subcategory wise data + Objective: Convert Object from one format to another. + eg: Govt JSON to Internal Data Structure Args: - subcategory_wise_data (dict): subcategory wise data - mapping (dict): subcategory to category mapping - with_subcategory (bool): include subcategory level data - - Returns: - dict: category wise data - - Example (with_subcategory=True): - { - "B2B, SEZ, DE": { - "B2B": data, - ... - } - ... - } - - Example (with_subcategory=False): - { - "B2B, SEZ, DE": data, - ... - } + data (dict): Data to be converted + default_data (dict, optional): Default Data to be added. Hardcoded values. + for_gov (bool, optional): If the data is to be converted to Govt JSON. Defaults to False. + else it will be converted to Internal Data Structure. + + Steps: + 1. Use key mapping to map the keys from one format to another. + 2. Use value formatters to format the values of the keys. + 3. Round values """ - category_wise_data = {} - for subcategory, category in self.mapping.items(): - if not subcategory_wise_data.get(subcategory.value): - continue + output = {} - category_wise_data.setdefault(category.value, []).extend( - subcategory_wise_data.get(subcategory.value, []) - ) + if default_data: + output.update(default_data) - return category_wise_data + key_mapping = self.KEY_MAPPING.copy() - def convert_to_gov_data_format( - self, internal_data: dict, company_gstin: str - ) -> dict: - """ - converts internal data format to Gov data format for all categories - """ + if for_gov: + key_mapping = self.reverse_dict(key_mapping) - output = {} - for category, mapper_class in self.CLASS_MAP.items(): - if not internal_data.get(category): + value_formatters = ( + self.value_formatters_for_gov + if for_gov + else self.value_formatters_for_internal + ) + + for old_key, new_key in key_mapping.items(): + invoice_data_value = data.get(old_key, "") + + if not for_gov and old_key == "flag": + continue + + if new_key in self.DISCARD_IF_ZERO_FIELDS and not invoice_data_value: continue - output[category] = mapper_class().convert_to_gov_data_format( - internal_data.get(category), company_gstin=company_gstin - ) + if not (invoice_data_value or invoice_data_value == 0): + # continue if value is None or empty object + continue + + value_formatter = value_formatters.get(old_key) + + if callable(value_formatter): + output[new_key] = value_formatter(invoice_data_value, data) + else: + output[new_key] = invoice_data_value + + if new_key in self.FLOAT_FIELDS: + output[new_key] = flt(output[new_key], 2) return output - def summarize_retsum_data( - self, - input_data, - ): - if not input_data: - return [] - - summarized_data = [] - total_values_keys = [ - "total_igst_amount", - "total_cgst_amount", - "total_sgst_amount", - "total_cess_amount", - "total_taxable_value", - ] - amended_data = {key: 0 for key in total_values_keys} - - input_data = {row.get("description"): row for row in input_data} - - def _sum(row): - return flt(sum([row.get(key, 0) for key in total_values_keys]), 2) - - for category, sub_categories in self.category_sub_category_mapping.items(): - category = category.value - if category not in input_data: - continue + # common utils - # compute total liability and total amended data - amended_category_data = input_data.get(f"{category} (Amended)", {}) - for key in total_values_keys: - amended_data[key] += amended_category_data.get(key, 0) + def reverse_dict(self, data): + return {v: k for k, v in data.items()} - # add category data - if _sum(input_data[category]) == 0: - continue + # common value formatters + def map_place_of_supply(self, pos, *args): + if pos.isnumeric(): + return f"{pos}-{self.STATE_NUMBERS.get(pos)}" - summarized_data.append({**input_data.get(category), "indent": 0}) + return pos.split("-")[0] - # add subcategory data - for sub_category in sub_categories: - sub_category = sub_category.value - if sub_category not in input_data: - continue + def update_totals(self, invoice, items): + """ + Update item totals to the invoice row + """ + total_data = self.TOTAL_DEFAULTS.copy() + + for item in items: + for field, value in item.items(): + total_field = f"total_{field}" - if _sum(input_data[sub_category]) == 0: + if total_field not in total_data: continue - summarized_data.append( - { - **input_data.get(sub_category), - "indent": 1, - "consider_in_total_taxable_value": ( - False - if sub_category - in self.subcategory_not_considered_in_total_taxable_value - else True - ), - "consider_in_total_tax": ( - False - if sub_category - in self.subcategory_not_considered_in_total_tax - else True - ), - } - ) - - # add total amendment liability - if _sum(amended_data) != 0: - summarized_data.extend( - [ - { - "description": "Net Liability from Amendments", - **amended_data, - "indent": 0, - "consider_in_total_taxable_value": True, - "consider_in_total_tax": True, - "no_of_records": 0, - } - ] - ) - - return summarized_data + invoice[total_field] = invoice.setdefault(total_field, 0) + value + + def set_total_defaults(self): + self.TOTAL_DEFAULTS = { + f"total_{key}": 0 for key in self.DEFAULT_ITEM_AMOUNTS.keys() + } diff --git a/india_compliance/gst_india/utils/itc_04/__init__.py b/india_compliance/gst_india/utils/itc_04/__init__.py index c344402ad..0e515f1b9 100644 --- a/india_compliance/gst_india/utils/itc_04/__init__.py +++ b/india_compliance/gst_india/utils/itc_04/__init__.py @@ -7,36 +7,65 @@ class GovJsonKey(Enum): """ TABLE5A = "table5A" - TABLE5B = "table5B" + STOCK_ENTRY = "m2jw" + + +class ITC04JsonKey(Enum): + """ + Categories / Keys as per Internal JSON file + """ + + TABLE5A = "Table 5A" + STOCK_ENTRY = "Stock Entry" class GovDataField(Enum): COMPANY_GSTIN = "ctin" JOB_WORKER_STATE_CODE = "jw_stcd" ITEMS = "items" - ORIGINAL_CAHLLAN_NUMBER = "o_chnum" + ORIGINAL_CHALLAN_NUMBER = "o_chnum" ORIGINAL_CHALLAN_DATE = "o_chdt" - CHALLAN_NUMBER = "jw2_chnum" - CHALLAN_DATE = "jw2_chdt" + JOB_WORK_CHALLAN_NUMBER = "jw2_chnum" + JOB_WORK_CHALLAN_DATE = "jw2_chdt" NATURE_OF_JOB = "nat_jw" UOM = "uqc" - QTY = "qty" + QUANTITY = "qty" DESCRIPTION = "desc" LOSS_UOM = "lwuqc" LOSS_QTY = "lwqty" + TAXABLE_VALUE = "txval" + GOODS_TYPE = "goods_ty" + IGST = "tx_i" + CGST = "tx_c" + SGST = "tx_s" + CESS_AMOUNT = "tx_cs" + FLAG = "flag" + + +class GovDataField_SE(Enum): + ITEMS = "itms" + ORIGINAL_CHALLAN_NUMBER = "chnum" + ORIGINAL_CHALLAN_DATE = "chdt" class ITC04_DataField(Enum): COMPANY_GSTIN = "company_gstin" JOB_WORKER_STATE_CODE = "jw_state_code" ITEMS = "items" - ORIGINAL_CAHLLAN_NUMBER = "original_challan_number" + ORIGINAL_CHALLAN_NUMBER = "original_challan_number" ORIGINAL_CHALLAN_DATE = "original_challan_date" - CHALLAN_NUMBER = "jw_challan_number" - CHALLAN_DATE = "jw_challan_date" + JOB_WORK_CHALLAN_NUMBER = "jw_challan_number" + JOB_WORK_CHALLAN_DATE = "jw_challan_date" NATURE_OF_JOB = "nature_of_job" UOM = "uom" - QTY = "qty" + QUANTITY = "qty" DESCRIPTION = "desc" LOSS_UOM = "loss_uom" LOSS_QTY = "loss_qty" + TAXABLE_VALUE = "taxable_value" + GOODS_TYPE = "goods_type" + IGST = "igst_rate" + CGST = "cgst_rate" + SGST = "sgst_rate" + CESS_AMOUNT = "cess_amount" + FLAG = "flag" diff --git a/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py b/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py index e50f78922..4e01b2fbe 100644 --- a/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py +++ b/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py @@ -1,9 +1,11 @@ -from india_compliance.gst_india.constants import STATE_NUMBERS, UOM_MAP -from india_compliance.gst_india.utils.gstr_mapper_utils import GSTRDataMapper +from india_compliance.gst_india.constants import UOM_MAP +from india_compliance.gst_india.utils.gstr_mapper_utils import GovDataMapper from india_compliance.gst_india.utils.itc_04 import ( GovDataField, + GovDataField_SE, GovJsonKey, ITC04_DataField, + ITC04JsonKey, ) ############################################################################################################ @@ -11,74 +13,34 @@ ############################################################################################################ -class GovDataMapper: +class ITC04DataMapper(GovDataMapper): """ GST Developer API Documentation for Returns - https://developer.gst.gov.in/apiportal/taxpayer/returns ITC-04 JSON format - https://developer.gst.gov.in/pages/apiportal/data/Returns/ITC04%20-%20Save/v1.2/ITC04%20-%20Save%20attributes.xlsx """ - KEY_MAPPING = {} - - def __init__(self): - self.value_formatters_for_internal = {} - self.value_formatters_for_gov = {} - self.STATE_NUMBERS = self.reverse_dict(STATE_NUMBERS) - - def format_data( - self, data: dict, default_data: dict = None, for_gov: bool = False - ) -> dict: - """ - Objective: Convert Object from one format to another. - eg: Govt JSON to Internal Data Structure - - Args: - data (dict): Data to be converted - default_data (dict, optional): Default Data to be added. Hardcoded values. - for_gov (bool, optional): If the data is to be converted to Govt JSON. Defaults to False. - else it will be converted to Internal Data Structure. - - Steps: - 1. Use key mapping to map the keys from one format to another. - 2. Use value formatters to format the values of the keys. - 3. Round values - """ - output = {} - - if default_data: - output.update(default_data) - - key_mapping = self.KEY_MAPPING.copy() - - if for_gov: - key_mapping = self.reverse_dict(key_mapping) - - value_formatters = ( - self.value_formatters_for_gov - if for_gov - else self.value_formatters_for_internal - ) - - for old_key, new_key in key_mapping.items(): - invoice_data_value = data.get(old_key, "") - - if not for_gov and old_key == "flag": - continue - - if not (invoice_data_value or invoice_data_value == 0): - # continue if value is None or empty object - continue - - value_formatter = value_formatters.get(old_key) + DEFAULT_ITEM_AMOUNTS = { + ITC04_DataField.TAXABLE_VALUE.value: 0, + ITC04_DataField.IGST.value: 0, + ITC04_DataField.CGST.value: 0, + ITC04_DataField.SGST.value: 0, + ITC04_DataField.CESS_AMOUNT.value: 0, + } - if callable(value_formatter): - output[new_key] = value_formatter(invoice_data_value) - else: - output[new_key] = invoice_data_value + FLOAT_FIELDS = { + GovDataField.TAXABLE_VALUE.value, + GovDataField.IGST.value, + GovDataField.CGST.value, + GovDataField.SGST.value, + GovDataField.CESS_AMOUNT.value, + GovDataField.QUANTITY.value, + } - return output + def __init__(self): + super().__init__() - def map_uom(self, uom): + def map_uom(self, uom, *args): uom = uom.upper() if "-" in uom: @@ -89,33 +51,25 @@ def map_uom(self, uom): return f"OTH-{UOM_MAP.get('OTH')}" - def map_place_of_supply(self, state_code): - if state_code.isnumeric(): - return f"{state_code}-{self.STATE_NUMBERS.get(state_code)}" - - return state_code.split("-")[0] - - def reverse_dict(self, data): - return {v: k for k, v in data.items()} - -class TABLE5A(GovDataMapper): - CATEGORY = GovJsonKey.TABLE5A.value +class TABLE5A(ITC04DataMapper): + CATEGORY = ITC04JsonKey.TABLE5A.value KEY_MAPPING = { GovDataField.COMPANY_GSTIN.value: ITC04_DataField.COMPANY_GSTIN.value, GovDataField.JOB_WORKER_STATE_CODE.value: ITC04_DataField.JOB_WORKER_STATE_CODE.value, GovDataField.ITEMS.value: ITC04_DataField.ITEMS.value, - GovDataField.ORIGINAL_CAHLLAN_NUMBER.value: ITC04_DataField.ORIGINAL_CAHLLAN_NUMBER.value, + GovDataField.ORIGINAL_CHALLAN_NUMBER.value: ITC04_DataField.ORIGINAL_CHALLAN_NUMBER.value, GovDataField.ORIGINAL_CHALLAN_DATE.value: ITC04_DataField.ORIGINAL_CHALLAN_DATE.value, - GovDataField.CHALLAN_NUMBER.value: ITC04_DataField.CHALLAN_NUMBER.value, - GovDataField.CHALLAN_DATE.value: ITC04_DataField.CHALLAN_DATE.value, + GovDataField.JOB_WORK_CHALLAN_NUMBER.value: ITC04_DataField.JOB_WORK_CHALLAN_NUMBER.value, + GovDataField.JOB_WORK_CHALLAN_DATE.value: ITC04_DataField.JOB_WORK_CHALLAN_DATE.value, GovDataField.NATURE_OF_JOB.value: ITC04_DataField.NATURE_OF_JOB.value, GovDataField.UOM.value: ITC04_DataField.UOM.value, - GovDataField.QTY.value: ITC04_DataField.QTY.value, + GovDataField.QUANTITY.value: ITC04_DataField.QUANTITY.value, GovDataField.DESCRIPTION.value: ITC04_DataField.DESCRIPTION.value, GovDataField.LOSS_UOM.value: ITC04_DataField.LOSS_UOM.value, GovDataField.LOSS_QTY.value: ITC04_DataField.LOSS_QTY.value, + GovDataField.FLAG.value: ITC04_DataField.FLAG.value, } def __init__(self): @@ -140,9 +94,14 @@ def convert_to_internal_data_format(self, input_data): for invoice in input_data: original_challan_number = invoice.get(GovDataField.ITEMS.value)[0].get( - GovDataField.ORIGINAL_CAHLLAN_NUMBER.value + GovDataField.ORIGINAL_CHALLAN_NUMBER.value + ) + job_work_challan_number = invoice.get(GovDataField.ITEMS.value)[0].get( + GovDataField.JOB_WORK_CHALLAN_NUMBER.value + ) + output[f"{original_challan_number} - {job_work_challan_number}"] = ( + self.format_data(invoice) ) - output[original_challan_number] = self.format_data(invoice) return {self.CATEGORY: output} @@ -156,7 +115,7 @@ def convert_to_gov_data_format(self, input_data, **kwargs): return output - def format_item_for_internal(self, items): + def format_item_for_internal(self, items, *args): return [ { **self.format_data(item), @@ -165,23 +124,124 @@ def format_item_for_internal(self, items): ] def format_item_for_gov(self, items, *args): + return [self.format_data(item, for_gov=True) for item in items] + + +class STOCK_ENTRY(ITC04DataMapper): + CATEGORY = ITC04JsonKey.STOCK_ENTRY.value + + KEY_MAPPING = { + GovDataField.COMPANY_GSTIN.value: ITC04_DataField.COMPANY_GSTIN.value, + GovDataField.JOB_WORKER_STATE_CODE.value: ITC04_DataField.JOB_WORKER_STATE_CODE.value, + GovDataField_SE.ITEMS.value: ITC04_DataField.ITEMS.value, + GovDataField_SE.ORIGINAL_CHALLAN_NUMBER.value: ITC04_DataField.ORIGINAL_CHALLAN_NUMBER.value, + GovDataField_SE.ORIGINAL_CHALLAN_DATE.value: ITC04_DataField.ORIGINAL_CHALLAN_DATE.value, + GovDataField.UOM.value: ITC04_DataField.UOM.value, + GovDataField.QUANTITY.value: ITC04_DataField.QUANTITY.value, + GovDataField.DESCRIPTION.value: ITC04_DataField.DESCRIPTION.value, + GovDataField.TAXABLE_VALUE.value: ITC04_DataField.TAXABLE_VALUE.value, + GovDataField.GOODS_TYPE.value: ITC04_DataField.GOODS_TYPE.value, + GovDataField.IGST.value: ITC04_DataField.IGST.value, + GovDataField.CGST.value: ITC04_DataField.CGST.value, + GovDataField.SGST.value: ITC04_DataField.SGST.value, + GovDataField.CESS_AMOUNT.value: ITC04_DataField.CESS_AMOUNT.value, + GovDataField.FLAG.value: ITC04_DataField.FLAG.value, + } + + def __init__(self): + super().__init__() + + self.value_formatters_for_internal = { + GovDataField_SE.ITEMS.value: self.format_item_for_internal, + GovDataField.UOM.value: self.map_uom, + GovDataField.JOB_WORKER_STATE_CODE.value: self.map_place_of_supply, + } + + self.value_formatters_for_gov = { + ITC04_DataField.ITEMS.value: self.format_item_for_gov, + ITC04_DataField.UOM.value: self.map_uom, + ITC04_DataField.JOB_WORKER_STATE_CODE.value: self.map_place_of_supply, + } + + def convert_to_internal_data_format(self, input_data): + output = {} + + for invoice in input_data: + original_challan_number = invoice.get( + GovDataField_SE.ORIGINAL_CHALLAN_NUMBER.value + ) + + invoice_level_data = self.format_data(invoice) + + self.update_totals( + invoice_level_data, + invoice_level_data.get(ITC04_DataField.ITEMS.value), + ) + output[str(original_challan_number)] = invoice_level_data + + return {self.CATEGORY: output} + + def convert_to_gov_data_format(self, input_data, **kwargs): + output = [] + + for invoice in input_data: + output.append(self.format_data(invoice, for_gov=True)) + + return output + + def format_item_for_internal(self, items, *args): return [ { - **self.format_data(item, for_gov=True), + **self.DEFAULT_ITEM_AMOUNTS.copy(), + **self.format_data(item), } for item in items ] + def format_item_for_gov(self, items, *args): + return [self.format_data(item, for_gov=True) for item in items] -class TABLE5B(TABLE5A): - CATEGORY = GovJsonKey.TABLE5B.value - def __init__(self): - super().__init__() +CLASS_MAP = { + GovJsonKey.TABLE5A.value: TABLE5A, + GovJsonKey.STOCK_ENTRY.value: STOCK_ENTRY, +} +CATEGORY_MAP = { + GovJsonKey.TABLE5A.value: ITC04JsonKey.TABLE5A.value, + GovJsonKey.STOCK_ENTRY.value: ITC04JsonKey.STOCK_ENTRY.value, +} -class ITC04DataMapper(GSTRDataMapper): - CLASS_MAP = { - GovJsonKey.TABLE5A.value: TABLE5A, - GovJsonKey.TABLE5B.value: TABLE5B, - } + +def convert_to_internal_data_format(gov_data): + """ + Converts Gov data format to internal data format for all categories + """ + output = {} + + for category, mapper_class in CLASS_MAP.items(): + if not gov_data.get(category): + continue + + output.update( + mapper_class().convert_to_internal_data_format(gov_data.get(category)) + ) + + return output + + +def convert_to_gov_data_format(internal_data: dict, company_gstin: str) -> dict: + """ + converts internal data format to Gov data format for all categories + """ + + output = {} + for category, mapper_class in CLASS_MAP.items(): + if not internal_data.get(CATEGORY_MAP.get(category)): + continue + + output[category] = mapper_class().convert_to_gov_data_format( + internal_data.get(CATEGORY_MAP.get(category)), company_gstin=company_gstin + ) + + return output From ef334f556f34fd3b6141b7d74ac025897ef1eaf5 Mon Sep 17 00:00:00 2001 From: Ninad1306 Date: Mon, 24 Jun 2024 18:07:26 +0530 Subject: [PATCH 04/10] refactor: avoid duplication --- .../gst_india/utils/gstr_mapper_utils.py | 6 ---- .../gst_india/utils/itc_04/itc_04_json_map.py | 35 ++++++------------- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/india_compliance/gst_india/utils/gstr_mapper_utils.py b/india_compliance/gst_india/utils/gstr_mapper_utils.py index 48171c454..819e57345 100644 --- a/india_compliance/gst_india/utils/gstr_mapper_utils.py +++ b/india_compliance/gst_india/utils/gstr_mapper_utils.py @@ -4,12 +4,6 @@ class GovDataMapper: - """ - GST Developer API Documentation for Returns - https://developer.gst.gov.in/apiportal/taxpayer/returns - - GSTR-1 JSON format - https://developer.gst.gov.in/pages/apiportal/data/Returns/GSTR1%20-%20Save%20GSTR1%20data/v4.0/GSTR1%20-%20Save%20GSTR1%20data%20attributes.xlsx - """ - KEY_MAPPING = {} FLOAT_FIELDS = {} DISCARD_IF_ZERO_FIELDS = {} diff --git a/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py b/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py index 4e01b2fbe..b22d463bf 100644 --- a/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py +++ b/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py @@ -51,6 +51,17 @@ def map_uom(self, uom, *args): return f"OTH-{UOM_MAP.get('OTH')}" + def convert_to_gov_data_format(self, input_data, **kwargs): + output = [] + + for invoice in input_data: + output.append(self.format_data(invoice, for_gov=True)) + + return output + + def format_item_for_gov(self, items, *args): + return [self.format_data(item, for_gov=True) for item in items] + class TABLE5A(ITC04DataMapper): CATEGORY = ITC04JsonKey.TABLE5A.value @@ -105,16 +116,6 @@ def convert_to_internal_data_format(self, input_data): return {self.CATEGORY: output} - def convert_to_gov_data_format(self, input_data, **kwargs): - output = [] - - for invoice in input_data: - formatted_data = self.format_data(invoice, for_gov=True) - - output.append(formatted_data) - - return output - def format_item_for_internal(self, items, *args): return [ { @@ -123,9 +124,6 @@ def format_item_for_internal(self, items, *args): for item in items ] - def format_item_for_gov(self, items, *args): - return [self.format_data(item, for_gov=True) for item in items] - class STOCK_ENTRY(ITC04DataMapper): CATEGORY = ITC04JsonKey.STOCK_ENTRY.value @@ -181,14 +179,6 @@ def convert_to_internal_data_format(self, input_data): return {self.CATEGORY: output} - def convert_to_gov_data_format(self, input_data, **kwargs): - output = [] - - for invoice in input_data: - output.append(self.format_data(invoice, for_gov=True)) - - return output - def format_item_for_internal(self, items, *args): return [ { @@ -198,9 +188,6 @@ def format_item_for_internal(self, items, *args): for item in items ] - def format_item_for_gov(self, items, *args): - return [self.format_data(item, for_gov=True) for item in items] - CLASS_MAP = { GovJsonKey.TABLE5A.value: TABLE5A, From c985870adc419a432aab9e568c03ec0d60706644 Mon Sep 17 00:00:00 2001 From: Ninad1306 Date: Tue, 25 Jun 2024 10:31:09 +0530 Subject: [PATCH 05/10] fix: redundant class removed --- .../gst_india/doctype/gstr_1_beta/gstr_1_export.py | 11 +++++++---- .../gst_india/doctype/gstr_1_log/gstr_1_log.py | 4 ++-- .../gst_india/utils/gstr_1/gstr_1_download.py | 8 +++++--- .../gst_india/utils/gstr_1/test_gstr_1_json_map.py | 8 ++------ 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/india_compliance/gst_india/doctype/gstr_1_beta/gstr_1_export.py b/india_compliance/gst_india/doctype/gstr_1_beta/gstr_1_export.py index 2bb1fe955..e0f098f30 100644 --- a/india_compliance/gst_india/doctype/gstr_1_beta/gstr_1_export.py +++ b/india_compliance/gst_india/doctype/gstr_1_beta/gstr_1_export.py @@ -21,7 +21,10 @@ GSTR1_ItemField, GSTR1_SubCategory, ) -from india_compliance.gst_india.utils.gstr_1.gstr_1_json_map import GSTR1DataMapper +from india_compliance.gst_india.utils.gstr_1.gstr_1_json_map import ( + convert_to_gov_data_format, + get_category_wise_data, +) class ExcelWidth(Enum): @@ -55,7 +58,7 @@ def process_data(self, input_data): 2. Format/Transform the data to match the Gov Excel format """ - category_wise_data = GSTR1DataMapper().get_category_wise_data(input_data) + category_wise_data = get_category_wise_data(input_data) processed_data = {} for category, data in category_wise_data.items(): @@ -1138,7 +1141,7 @@ def __init__(self, company_gstin, month_or_quarter, year): self.summary = gstr1_log.load_data("reconcile_summary")["reconcile_summary"] data = gstr1_log.load_data("reconcile")["reconcile"] - self.data = GSTR1DataMapper().get_category_wise_data(data) + self.data = get_category_wise_data(data) def export_data(self): excel = ExcelExporter() @@ -2077,7 +2080,7 @@ def download_gstr_1_json( "data": { "gstin": company_gstin, "fp": period, - **GSTR1DataMapper().convert_to_gov_data_format(data, company_gstin), + **convert_to_gov_data_format(data, company_gstin), }, "filename": f"GSTR-1-Gov-{company_gstin}-{period}.json", } diff --git a/india_compliance/gst_india/doctype/gstr_1_log/gstr_1_log.py b/india_compliance/gst_india/doctype/gstr_1_log/gstr_1_log.py index 6b3279da9..8bd15c7a6 100644 --- a/india_compliance/gst_india/doctype/gstr_1_log/gstr_1_log.py +++ b/india_compliance/gst_india/doctype/gstr_1_log/gstr_1_log.py @@ -22,7 +22,7 @@ ) from india_compliance.gst_india.utils.gstr_1.gstr_1_json_map import ( GSTR1BooksData, - GSTR1DataMapper, + summarize_retsum_data, ) from india_compliance.gst_india.utils.gstr_utils import request_otp @@ -41,7 +41,7 @@ def get_summarized_data(self, data, is_filed=False): Helper function to summarize data for each sub-category """ if is_filed and data.get("summary"): - return GSTR1DataMapper().summarize_retsum_data(data.get("summary")) + return summarize_retsum_data(data.get("summary")) subcategory_summary = self.get_subcategory_summary(data) diff --git a/india_compliance/gst_india/utils/gstr_1/gstr_1_download.py b/india_compliance/gst_india/utils/gstr_1/gstr_1_download.py index 5d27d1b39..94a4d2102 100644 --- a/india_compliance/gst_india/utils/gstr_1/gstr_1_download.py +++ b/india_compliance/gst_india/utils/gstr_1/gstr_1_download.py @@ -6,7 +6,9 @@ from india_compliance.gst_india.doctype.gstr_import_log.gstr_import_log import ( create_import_log, ) -from india_compliance.gst_india.utils.gstr_1.gstr_1_json_map import GSTR1DataMapper +from india_compliance.gst_india.utils.gstr_1.gstr_1_json_map import ( + convert_to_internal_data_format, +) UNFILED_ACTIONS = [ "B2B", @@ -75,7 +77,7 @@ def download_gstr1_json_data(gstr1_log): json_data.update(response) - mapped_data = GSTR1DataMapper().convert_to_internal_data_format(json_data) + mapped_data = convert_to_internal_data_format(json_data) gstr1_log.update_json_for(data_field, mapped_data, reset_reconcile=True) if is_queued: @@ -107,7 +109,7 @@ def save_gstr_1(gstin, return_period, json_data, return_type): title=_("Invalid Response Received."), ) - mapped_data = GSTR1DataMapper().convert_to_internal_data_format(json_data) + mapped_data = convert_to_internal_data_format(json_data) gstr1_log = frappe.get_doc("GSTR-1 Log", f"{return_period}-{gstin}") gstr1_log.update_json_for(data_field, mapped_data, overwrite=False) diff --git a/india_compliance/gst_india/utils/gstr_1/test_gstr_1_json_map.py b/india_compliance/gst_india/utils/gstr_1/test_gstr_1_json_map.py index c5f1d9ec1..d9da34654 100644 --- a/india_compliance/gst_india/utils/gstr_1/test_gstr_1_json_map.py +++ b/india_compliance/gst_india/utils/gstr_1/test_gstr_1_json_map.py @@ -23,8 +23,8 @@ SUPECOM, TXPD, Exports, - GSTR1DataMapper, NilRated, + get_category_wise_data, ) @@ -37,11 +37,7 @@ def normalize_data(data): def process_mapped_data(data): - return list( - GSTR1DataMapper() - .get_category_wise_data(normalize_data(copy.deepcopy(data))) - .values() - )[0] + return list(get_category_wise_data(normalize_data(copy.deepcopy(data))).values())[0] class TestB2B(FrappeTestCase): From ba97635ee0d27df8d5a43eb1fbb3b0dec0999d7a Mon Sep 17 00:00:00 2001 From: Ninad1306 Date: Tue, 25 Jun 2024 12:09:22 +0530 Subject: [PATCH 06/10] fix: display challan_no at invoice level --- .../gst_india/utils/itc_04/itc_04_json_map.py | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py b/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py index b22d463bf..ee6a1aa38 100644 --- a/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py +++ b/india_compliance/gst_india/utils/itc_04/itc_04_json_map.py @@ -35,6 +35,7 @@ class ITC04DataMapper(GovDataMapper): GovDataField.SGST.value, GovDataField.CESS_AMOUNT.value, GovDataField.QUANTITY.value, + GovDataField.LOSS_QTY.value, } def __init__(self): @@ -70,9 +71,7 @@ class TABLE5A(ITC04DataMapper): GovDataField.COMPANY_GSTIN.value: ITC04_DataField.COMPANY_GSTIN.value, GovDataField.JOB_WORKER_STATE_CODE.value: ITC04_DataField.JOB_WORKER_STATE_CODE.value, GovDataField.ITEMS.value: ITC04_DataField.ITEMS.value, - GovDataField.ORIGINAL_CHALLAN_NUMBER.value: ITC04_DataField.ORIGINAL_CHALLAN_NUMBER.value, GovDataField.ORIGINAL_CHALLAN_DATE.value: ITC04_DataField.ORIGINAL_CHALLAN_DATE.value, - GovDataField.JOB_WORK_CHALLAN_NUMBER.value: ITC04_DataField.JOB_WORK_CHALLAN_NUMBER.value, GovDataField.JOB_WORK_CHALLAN_DATE.value: ITC04_DataField.JOB_WORK_CHALLAN_DATE.value, GovDataField.NATURE_OF_JOB.value: ITC04_DataField.NATURE_OF_JOB.value, GovDataField.UOM.value: ITC04_DataField.UOM.value, @@ -111,11 +110,31 @@ def convert_to_internal_data_format(self, input_data): GovDataField.JOB_WORK_CHALLAN_NUMBER.value ) output[f"{original_challan_number} - {job_work_challan_number}"] = ( - self.format_data(invoice) + self.format_data( + invoice, + { + ITC04_DataField.ORIGINAL_CHALLAN_NUMBER.value: original_challan_number, + ITC04_DataField.JOB_WORK_CHALLAN_NUMBER.value: job_work_challan_number, + }, + ) ) return {self.CATEGORY: output} + def convert_to_gov_data_format(self, input_data, **kwargs): + output = [] + + for invoice in input_data: + self.original_challan_number = invoice.get( + ITC04_DataField.ORIGINAL_CHALLAN_NUMBER.value + ) + self.job_work_challan_number = invoice.get( + ITC04_DataField.JOB_WORK_CHALLAN_NUMBER.value + ) + output.append(self.format_data(invoice, for_gov=True)) + + return output + def format_item_for_internal(self, items, *args): return [ { @@ -124,6 +143,19 @@ def format_item_for_internal(self, items, *args): for item in items ] + def format_item_for_gov(self, items, *args): + return [ + self.format_data( + item, + { + GovDataField.ORIGINAL_CHALLAN_NUMBER.value: self.original_challan_number, + GovDataField.JOB_WORK_CHALLAN_NUMBER.value: self.job_work_challan_number, + }, + for_gov=True, + ) + for item in items + ] + class STOCK_ENTRY(ITC04DataMapper): CATEGORY = ITC04JsonKey.STOCK_ENTRY.value From 337177b8fecbb235d8e4d1e015239722021be764 Mon Sep 17 00:00:00 2001 From: Ninad Parikh Date: Wed, 2 Oct 2024 11:48:45 +0530 Subject: [PATCH 07/10] fix: json download for itc-04 return --- .../gst_job_work_stock_movement.js | 63 +++++++ .../gst_india/utils/itc_04/itc_04_export.py | 169 ++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 india_compliance/gst_india/utils/itc_04/itc_04_export.py diff --git a/india_compliance/gst_india/report/gst_job_work_stock_movement/gst_job_work_stock_movement.js b/india_compliance/gst_india/report/gst_job_work_stock_movement/gst_job_work_stock_movement.js index 85eea61ee..a70075190 100644 --- a/india_compliance/gst_india/report/gst_job_work_stock_movement/gst_job_work_stock_movement.js +++ b/india_compliance/gst_india/report/gst_job_work_stock_movement/gst_job_work_stock_movement.js @@ -57,4 +57,67 @@ frappe.query_reports["GST Job Work Stock Movement"] = { reqd: 1, }, ], + onload: function (query_report) { + query_report.page.add_inner_button(__("Export JSON"), function () { + this.dialog = new frappe.ui.Dialog({ + title: __("Export JSON"), + fields: [ + { + fieldname: "year", + label: __("Year"), + fieldtype: "Select", + options: get_options_for_year(), + onchange: () => { + if (this.dialog.get_value("year") === "2017") { + this.dialog.fields_dict.period.df.options = [ + "Jul - Sep", + "Oct - Dec", + "Jan - Mar", + ]; + this.dialog.refresh(); + } + }, + }, + { + fieldname: "period", + label: __("Return Filing Period"), + fieldtype: "Select", + options: ["Apr - Jun", "Jul - Sep", "Oct - Dec", "Jan - Mar"], + }, + ], + primary_action_label: "Export JSON", + primary_action: () => { + frappe.call({ + method: "india_compliance.gst_india.utils.itc_04.itc_04_export.download_itc_04_json", + args: { + company: frappe.query_report.get_filter_value("company"), + company_gstin: + frappe.query_report.get_filter_value("company_gstin"), + period: this.dialog.get_value("period"), + year: this.dialog.get_value("year"), + }, + callback: r => { + this.dialog.hide(); + india_compliance.trigger_file_download( + JSON.stringify(r.message.data), + r.message.filename + ); + }, + }); + }, + }); + + dialog.show(); + }); + query_report.refresh(); + }, }; + +function get_options_for_year() { + const today = new Date(); + const current_year = today.getFullYear(); + const start_year = 2017; + const year_range = current_year - start_year + 1; + let options = Array.from({ length: year_range }, (_, index) => start_year + index); + return options.reverse().map(year => year.toString()); +} diff --git a/india_compliance/gst_india/utils/itc_04/itc_04_export.py b/india_compliance/gst_india/utils/itc_04/itc_04_export.py new file mode 100644 index 000000000..a7bb311bd --- /dev/null +++ b/india_compliance/gst_india/utils/itc_04/itc_04_export.py @@ -0,0 +1,169 @@ +import frappe +from frappe.utils import cint, format_date, get_date_str, get_first_day, get_last_day + +from india_compliance.gst_india.constants import UOM_MAP +from india_compliance.gst_india.doctype.gst_return_log.generate_gstr_1 import ( + GenerateGSTR1, +) +from india_compliance.gst_india.utils import get_month_or_quarter_dict +from india_compliance.gst_india.utils.itc_04.itc_04_data import ITC04Query +from india_compliance.gst_india.utils.itc_04.itc_04_json_map import ( + convert_to_gov_data_format, +) + + +@frappe.whitelist() +def download_itc_04_json( + company, + company_gstin, + period, + year, +): + frappe.has_permission("GST Job Work Stock Movement", "export", throw=True) + + filters = get_filters( + company, + company_gstin, + period, + year, + ) + ret_period = get_return_period(period, year) + data = get_data(filters) + + GenerateGSTR1().normalize_data(data) + + return { + "data": { + "gstin": company_gstin, + "fp": ret_period, + **convert_to_gov_data_format(data, company_gstin), + }, + "filename": f"ITC-04-Gov-{company_gstin}-{ret_period}.json", + } + + +def get_filters( + company, + company_gstin, + period, + year, +): + filters = {} + quarter_no = get_month_or_quarter_dict().get(period) + filters["from_date"] = get_first_day(f"{cint(year)}-{quarter_no[0]}-01") + filters["to_date"] = get_last_day(f"{cint(year)}-{quarter_no[1]}-01") + filters["company_gstin"] = company_gstin + filters["company"] = company + + return filters + + +def get_data(filters): + itc04 = ITC04Query(filters) + + table_4_data = itc04.get_query_table_4_se().run( + as_dict=True + ) + itc04.get_query_table_4_sr().run(as_dict=True) + + table_5a_data = itc04.get_query_table_5A_se().run( + as_dict=True + ) + itc04.get_query_table_5A_sr().run(as_dict=True) + + return { + "Stock Entry": process_table_4_data(table_4_data), + "Table 5A": process_table_5a_data(table_5a_data), + } + + +def process_table_4_data(invoice_data): + def create_item(invoice, uom): + return { + "taxable_value": invoice.taxable_value, + "igst_rate": invoice.igst_amount, + "cgst_rate": invoice.cgst_amount, + "sgst_rate": invoice.sgst_amount, + "cess_amount": invoice.total_cess_amount, + "uom": f"{uom}-{UOM_MAP[uom]}", + "qty": invoice.qty, + "desc": "", + "goods_type": "7b", + } + + res = {} + + for invoice in invoice_data: + key = invoice.invoice_no + uom = invoice.uom.upper() + challan_date = format_date(get_date_str(invoice.posting_date), "dd-mm-yyyy") + + if key not in res: + res[key] = { + "jw_state_code": invoice.place_of_supply, + "flag": "", + "items": [create_item(invoice, uom)], + "original_challan_number": invoice.invoice_no, + "original_challan_date": challan_date, + "total_taxable_value": invoice.taxable_value, + "total_igst_rate": invoice.igst_amount, + "total_cgst_rate": invoice.cgst_amount, + "total_sgst_rate": invoice.sgst_amount, + "total_cess_amount": invoice.total_cess_amount, + } + else: + current_invoice = res[key] + current_invoice["total_taxable_value"] += invoice.taxable_value + current_invoice["total_igst_rate"] += invoice.igst_amount + current_invoice["total_cgst_rate"] += invoice.cgst_amount + current_invoice["total_sgst_rate"] += invoice.sgst_amount + current_invoice["total_cess_amount"] += invoice.total_cess_amount + current_invoice["items"].append(create_item(invoice, uom)) + + return res + + +def process_table_5a_data(invoice_data): + def create_item(invoice, uom, jw_challan_date, challan_date): + return { + "original_challan_date": challan_date, + "jw_challan_date": jw_challan_date, + "nature_of_job": "Work", + "uom": f"{uom}-{UOM_MAP[uom]}", + "qty": invoice.qty, + "desc": "", + } + + res = {} + + for invoice in invoice_data: + key = f"{invoice.original_challan_no} - {invoice.invoice_no}" + uom = invoice.uom.upper() + + jw_challan_date = format_date(get_date_str(invoice.posting_date), "dd-mm-yyyy") + challan_date = format_date( + get_date_str(invoice.original_challan_date), "dd-mm-yyyy" + ) + + if key not in res: + res[key] = { + "original_challan_number": invoice.original_challan_no, + "jw_challan_number": invoice.invoice_no, + "company_gstin": invoice.company_gstin, + "jw_state_code": invoice.place_of_supply, + "flag": "", + "items": [create_item(invoice, uom, jw_challan_date, challan_date)], + } + else: + res[key]["items"].append( + create_item(invoice, uom, jw_challan_date, challan_date) + ) + + return res + + +def get_return_period(month_or_quarter, year): + return { + "Apr - Jun": "13", + "Jul - Sep": "14", + "Oct - Dec": "15", + "Jan - Mar": "16", + }.get(month_or_quarter) + str(year) From aeabe5e49476fff51a3c1d8687c9534c1342ad75 Mon Sep 17 00:00:00 2001 From: Ninad Parikh Date: Wed, 2 Oct 2024 11:49:07 +0530 Subject: [PATCH 08/10] fix: query for original challan date --- .../gst_india/utils/itc_04/itc_04_data.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/india_compliance/gst_india/utils/itc_04/itc_04_data.py b/india_compliance/gst_india/utils/itc_04/itc_04_data.py index fb259a7c3..afb856cb3 100644 --- a/india_compliance/gst_india/utils/itc_04/itc_04_data.py +++ b/india_compliance/gst_india/utils/itc_04/itc_04_data.py @@ -152,6 +152,7 @@ def get_base_query_table_5A(self, doc, doc_item, ref_doc): IfNull(doc_item.gst_treatment, "Not Defined").as_("gst_treatment"), ref_doc.link_doctype.as_("original_challan_invoice_type"), IfNull(ref_doc.link_name, "").as_("original_challan_no"), + self.get_challan_date_query(ref_doc).as_("original_challan_date"), ) .where(doc.docstatus == 1) .orderby(doc.name, order=Order.desc) @@ -222,3 +223,14 @@ def get_query_with_common_filters(self, query, doc): query = query.where(Date(doc.posting_date) <= getdate(self.filters.to_date)) return query + + def get_challan_date_query(self, ref_doc): + se_doc = frappe.qb.DocType("Stock Entry") + sr_doc = frappe.qb.DocType("Subcontracting Receipt") + + return frappe.qb.from_(se_doc).select(se_doc.posting_date).where( + (se_doc.name == ref_doc.link_name) & (ref_doc.link_doctype == "Stock Entry") + ) + frappe.qb.from_(sr_doc).select(sr_doc.posting_date).where( + (sr_doc.name == ref_doc.link_name) + & (ref_doc.link_doctype == "Subcontracting Receipt") + ) From ae960b455a641fa82b8432a1dd28496ecffdec93 Mon Sep 17 00:00:00 2001 From: Ninad Parikh Date: Wed, 2 Oct 2024 12:23:54 +0530 Subject: [PATCH 09/10] fix: update query for job worker challan no --- india_compliance/gst_india/utils/itc_04/itc_04_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/india_compliance/gst_india/utils/itc_04/itc_04_data.py b/india_compliance/gst_india/utils/itc_04/itc_04_data.py index afb856cb3..543c29878 100644 --- a/india_compliance/gst_india/utils/itc_04/itc_04_data.py +++ b/india_compliance/gst_india/utils/itc_04/itc_04_data.py @@ -171,6 +171,7 @@ def get_query_table_5A_se(self): query = ( self.get_base_query_table_5A(self.se, self.se_item, self.ref_doc) .select( + IfNull(self.se.name, "").as_("invoice_no"), self.se_item.uom, self.se.bill_to_gstin.as_("supplier_gstin"), self.se.bill_from_gstin.as_("company_gstin"), @@ -195,6 +196,7 @@ def get_query_table_5A_sr(self): query = ( self.get_base_query_table_5A(self.sr, self.sr_item, self.ref_doc) .select( + IfNull(self.sr.supplier_delivery_note, "").as_("invoice_no"), self.sr_item.stock_uom.as_("uom"), self.sr.company_gstin, self.sr.supplier_gstin, From 9544f64f635b037044c976c3f8ca5bdefb048ea6 Mon Sep 17 00:00:00 2001 From: Ninad Parikh Date: Wed, 2 Oct 2024 12:24:35 +0530 Subject: [PATCH 10/10] fix: add description in query --- india_compliance/gst_india/utils/itc_04/itc_04_data.py | 3 ++- india_compliance/gst_india/utils/itc_04/itc_04_export.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/india_compliance/gst_india/utils/itc_04/itc_04_data.py b/india_compliance/gst_india/utils/itc_04/itc_04_data.py index 543c29878..50456916b 100644 --- a/india_compliance/gst_india/utils/itc_04/itc_04_data.py +++ b/india_compliance/gst_india/utils/itc_04/itc_04_data.py @@ -43,6 +43,7 @@ def get_base_query_table_4(self, doc, doc_item): doc.is_return, IfNull(doc.place_of_supply, "").as_("place_of_supply"), doc.base_grand_total.as_("invoice_total"), + IfNull(doc_item.description, "").as_("description"), IfNull(doc_item.gst_treatment, "Not Defined").as_("gst_treatment"), (doc_item.cgst_rate + doc_item.sgst_rate + doc_item.igst_rate).as_( "gst_rate" @@ -142,9 +143,9 @@ def get_base_query_table_5A(self, doc, doc_item, ref_doc): .select( IfNull(doc_item.item_code, doc_item.item_name).as_("item_code"), doc_item.qty, + IfNull(doc_item.description, "").as_("description"), doc_item.gst_hsn_code, IfNull(doc.supplier, "").as_("supplier"), - IfNull(doc.name, "").as_("invoice_no"), doc.posting_date, doc.is_return, IfNull(doc.place_of_supply, "").as_("place_of_supply"), diff --git a/india_compliance/gst_india/utils/itc_04/itc_04_export.py b/india_compliance/gst_india/utils/itc_04/itc_04_export.py index a7bb311bd..5bfe55efe 100644 --- a/india_compliance/gst_india/utils/itc_04/itc_04_export.py +++ b/india_compliance/gst_india/utils/itc_04/itc_04_export.py @@ -85,7 +85,7 @@ def create_item(invoice, uom): "cess_amount": invoice.total_cess_amount, "uom": f"{uom}-{UOM_MAP[uom]}", "qty": invoice.qty, - "desc": "", + "desc": invoice.description, "goods_type": "7b", } @@ -129,7 +129,7 @@ def create_item(invoice, uom, jw_challan_date, challan_date): "nature_of_job": "Work", "uom": f"{uom}-{UOM_MAP[uom]}", "qty": invoice.qty, - "desc": "", + "desc": invoice.description, } res = {}