From 1104c88e4844216054e2f2ddb030390354b280f9 Mon Sep 17 00:00:00 2001 From: Marc Bernard <59966492+mbtools@users.noreply.github.com> Date: Tue, 23 Mar 2021 06:40:09 +0100 Subject: [PATCH] Data deserialize (#4654) * Data deserialize Continuing on #3441 - Fixes issue with namespaced tables - Adds deserialize of table content during pull but does NOT persist changes to database `zcl_abapgit_repo->deserialize` now contains the following call to deserialize "data": ```abap zcl_abapgit_data_factory=>get_deserializer( )->deserialize( ii_config = get_data_config( ) it_files = get_files_remote( ) iv_persist = abap_false ). "< --- src/data/zcl_abapgit_data_config.clas.abap | 20 ++- .../zcl_abapgit_data_deserializer.clas.abap | 143 ++++++++++++++++-- .../zcl_abapgit_data_serializer.clas.abap | 33 ++-- src/data/zcl_abapgit_data_utils.clas.abap | 13 +- ...l_abapgit_data_utils.clas.testclasses.abap | 34 +++++ src/data/zcl_abapgit_data_utils.clas.xml | 1 + src/data/zif_abapgit_data_config.intf.abap | 2 + src/data/zif_abapgit_data_config.intf.xml | 2 +- .../zif_abapgit_data_deserializer.intf.abap | 6 +- .../zif_abapgit_data_deserializer.intf.xml | 2 +- src/data/zif_abapgit_data_serializer.intf.xml | 2 +- src/repo/zcl_abapgit_repo.clas.abap | 12 +- 12 files changed, 227 insertions(+), 43 deletions(-) create mode 100644 src/data/zcl_abapgit_data_utils.clas.testclasses.abap diff --git a/src/data/zcl_abapgit_data_config.clas.abap b/src/data/zcl_abapgit_data_config.clas.abap index a3842c03c..c98404a00 100644 --- a/src/data/zcl_abapgit_data_config.clas.abap +++ b/src/data/zcl_abapgit_data_config.clas.abap @@ -6,11 +6,9 @@ CLASS zcl_abapgit_data_config DEFINITION PUBLIC SECTION. INTERFACES zif_abapgit_data_config . - PROTECTED SECTION. PRIVATE SECTION. - CONSTANTS c_extension TYPE string VALUE '.config.json'. DATA mt_config TYPE zif_abapgit_data_config=>ty_config_tt . METHODS dump @@ -24,7 +22,7 @@ ENDCLASS. -CLASS ZCL_ABAPGIT_DATA_CONFIG IMPLEMENTATION. +CLASS zcl_abapgit_data_config IMPLEMENTATION. METHOD dump. @@ -32,12 +30,11 @@ CLASS ZCL_ABAPGIT_DATA_CONFIG IMPLEMENTATION. DATA lo_ajson TYPE REF TO zcl_abapgit_ajson. DATA lx_ajson TYPE REF TO zcx_abapgit_ajson_error. - TRY. lo_ajson = zcl_abapgit_ajson=>create_empty( ). lo_ajson->zif_abapgit_ajson_writer~set( iv_path = '/' - iv_val = is_config ). + iv_val = is_config ). rv_json = zcl_abapgit_convert=>string_to_xstring_utf8( lo_ajson->stringify( 2 ) ). CATCH zcx_abapgit_ajson_error INTO lx_ajson. zcx_abapgit_exception=>raise( lx_ajson->get_text( ) ). @@ -48,8 +45,8 @@ CLASS ZCL_ABAPGIT_DATA_CONFIG IMPLEMENTATION. METHOD zif_abapgit_data_config~add_config. - ASSERT NOT is_config-type IS INITIAL. - ASSERT NOT is_config-name IS INITIAL. + ASSERT is_config-type IS NOT INITIAL. + ASSERT is_config-name IS NOT INITIAL. ASSERT is_config-name = to_upper( is_config-name ). INSERT is_config INTO TABLE mt_config. @@ -69,7 +66,7 @@ CLASS ZCL_ABAPGIT_DATA_CONFIG IMPLEMENTATION. CLEAR mt_config. LOOP AT it_files INTO ls_file WHERE path = zif_abapgit_data_config=>c_default_path - AND filename CP |*{ c_extension }|. + AND filename CP |*.{ zif_abapgit_data_config=>c_config }.{ zif_abapgit_data_config=>c_default_format }|. TRY. lo_ajson = zcl_abapgit_ajson=>parse( zcl_abapgit_convert=>xstring_to_string_utf8( ls_file-data ) ). lo_ajson->zif_abapgit_ajson_reader~to_abap( IMPORTING ev_container = ls_config ). @@ -90,8 +87,8 @@ CLASS ZCL_ABAPGIT_DATA_CONFIG IMPLEMENTATION. METHOD zif_abapgit_data_config~remove_config. - ASSERT NOT is_config-type IS INITIAL. - ASSERT NOT is_config-name IS INITIAL. + ASSERT is_config-type IS NOT INITIAL. + ASSERT is_config-name IS NOT INITIAL. ASSERT is_config-name = to_upper( is_config-name ). DELETE mt_config WHERE name = is_config-name AND type = is_config-type. @@ -110,9 +107,10 @@ CLASS ZCL_ABAPGIT_DATA_CONFIG IMPLEMENTATION. ls_file-path = zif_abapgit_data_config=>c_default_path. LOOP AT mt_config INTO ls_config. - ls_file-filename = to_lower( |{ ls_config-name }{ c_extension }| ). ls_file-data = dump( ls_config ). ls_file-sha1 = zcl_abapgit_hash=>sha1_blob( ls_file-data ). + ls_config-type = zif_abapgit_data_config=>c_config. + ls_file-filename = zcl_abapgit_data_utils=>build_filename( ls_config ). APPEND ls_file TO rt_files. ENDLOOP. diff --git a/src/data/zcl_abapgit_data_deserializer.clas.abap b/src/data/zcl_abapgit_data_deserializer.clas.abap index 126b388cc..f8822842c 100644 --- a/src/data/zcl_abapgit_data_deserializer.clas.abap +++ b/src/data/zcl_abapgit_data_deserializer.clas.abap @@ -6,28 +6,53 @@ CLASS zcl_abapgit_data_deserializer DEFINITION PUBLIC SECTION. INTERFACES zif_abapgit_data_deserializer . + PROTECTED SECTION. + PRIVATE SECTION. - METHODS read_json + METHODS convert_json_to_itab IMPORTING !is_file TYPE zif_abapgit_definitions=>ty_file !ir_data TYPE REF TO data RAISING zcx_abapgit_exception . + METHODS preview_database_changes + IMPORTING + !iv_name TYPE tadir-obj_name + !it_where TYPE string_table + !ir_data TYPE REF TO data + RETURNING + VALUE(rs_result) TYPE zif_abapgit_data_deserializer=>ty_result + RAISING + zcx_abapgit_exception . + METHODS write_database_table + IMPORTING + !iv_name TYPE tadir-obj_name + !ir_del TYPE REF TO data + !ir_ins TYPE REF TO data + RAISING + zcx_abapgit_exception . + METHODS read_database_table + IMPORTING + !iv_name TYPE tadir-obj_name + !it_where TYPE string_table + RETURNING + VALUE(rr_data) TYPE REF TO data . + ENDCLASS. -CLASS ZCL_ABAPGIT_DATA_DESERIALIZER IMPLEMENTATION. +CLASS zcl_abapgit_data_deserializer IMPLEMENTATION. - METHOD read_json. + METHOD convert_json_to_itab. DATA lo_ajson TYPE REF TO zcl_abapgit_ajson. DATA lx_ajson TYPE REF TO zcx_abapgit_ajson_error. - FIELD-SYMBOLS TYPE ANY TABLE. + FIELD-SYMBOLS TYPE ANY TABLE. ASSIGN ir_data->* TO . @@ -41,9 +66,104 @@ CLASS ZCL_ABAPGIT_DATA_DESERIALIZER IMPLEMENTATION. ENDMETHOD. + METHOD preview_database_changes. + +* method currently distinguishes between records be deleted and inserted (comparison of complete record) +* to-do: compare records based on database key of table to determine updates to existing records + + DATA lr_data TYPE REF TO data. + + FIELD-SYMBOLS TYPE ANY TABLE. + FIELD-SYMBOLS TYPE ANY TABLE. + FIELD-SYMBOLS TYPE any. + FIELD-SYMBOLS TYPE any. + FIELD-SYMBOLS TYPE ANY TABLE. + FIELD-SYMBOLS TYPE ANY TABLE. + + lr_data = read_database_table( + iv_name = iv_name + it_where = it_where ). + + ASSIGN lr_data->* TO . + ASSIGN ir_data->* TO . + + rs_result-table = iv_name. + rs_result-deletes = zcl_abapgit_data_utils=>build_table_itab( iv_name ). + rs_result-inserts = zcl_abapgit_data_utils=>build_table_itab( iv_name ). + ASSIGN rs_result-deletes->* TO . + ASSIGN rs_result-inserts->* TO . + + = . + = . + + " Remove identical records + LOOP AT ASSIGNING . + READ TABLE ASSIGNING FROM . + IF sy-subrc = 0. + DELETE TABLE FROM . + DELETE TABLE FROM . + ENDIF. + ENDLOOP. + + ENDMETHOD. + + + METHOD read_database_table. + + DATA lv_where LIKE LINE OF it_where. + + FIELD-SYMBOLS TYPE ANY TABLE. + + rr_data = zcl_abapgit_data_utils=>build_table_itab( iv_name ). + ASSIGN rr_data->* TO . + + LOOP AT it_where INTO lv_where. + SELECT * FROM (iv_name) APPENDING TABLE WHERE (lv_where). + ENDLOOP. + IF lines( it_where ) = 0. + SELECT * FROM (iv_name) INTO TABLE . + ENDIF. + + ENDMETHOD. + + + METHOD write_database_table. + + FIELD-SYMBOLS TYPE ANY TABLE. + FIELD-SYMBOLS TYPE ANY TABLE. + + ASSIGN ir_del->* TO . + ASSIGN ir_ins->* TO . + + IF lines( ) > 0. + DELETE (iv_name) FROM TABLE . + IF sy-subrc <> 0. + zcx_abapgit_exception=>raise( |Error deleting { lines( ) } records from table { iv_name }| ). + ENDIF. + ENDIF. + + IF lines( ) > 0. + INSERT (iv_name) FROM TABLE . + IF sy-subrc <> 0. + zcx_abapgit_exception=>raise( |Error inserting { lines( ) } records into table { iv_name }| ). + ENDIF. + ENDIF. + + ENDMETHOD. + + METHOD zif_abapgit_data_deserializer~actualize. -* todo, this method will update the database +* this method updates the database + + DATA ls_result LIKE LINE OF it_result. + + LOOP AT it_result INTO ls_result. + write_database_table( + iv_name = ls_result-table + ir_del = ls_result-deletes + ir_ins = ls_result-inserts ). + ENDLOOP. ENDMETHOD. @@ -56,7 +176,7 @@ CLASS ZCL_ABAPGIT_DATA_DESERIALIZER IMPLEMENTATION. DATA ls_config LIKE LINE OF lt_configs. DATA lr_data TYPE REF TO data. DATA ls_file LIKE LINE OF it_files. - + DATA ls_result LIKE LINE OF rt_result. lt_configs = ii_config->get_configs( ). @@ -68,12 +188,17 @@ CLASS ZCL_ABAPGIT_DATA_DESERIALIZER IMPLEMENTATION. path = zif_abapgit_data_config=>c_default_path filename = zcl_abapgit_data_utils=>build_filename( ls_config ). IF sy-subrc = 0. - read_json( + convert_json_to_itab( ir_data = lr_data is_file = ls_file ). - ENDIF. -* todo + ls_result = preview_database_changes( + iv_name = ls_config-name + it_where = ls_config-where + ir_data = lr_data ). + + INSERT ls_result INTO TABLE rt_result. + ENDIF. ENDLOOP. diff --git a/src/data/zcl_abapgit_data_serializer.clas.abap b/src/data/zcl_abapgit_data_serializer.clas.abap index 553de2fb0..9267a5a19 100644 --- a/src/data/zcl_abapgit_data_serializer.clas.abap +++ b/src/data/zcl_abapgit_data_serializer.clas.abap @@ -6,9 +6,14 @@ CLASS zcl_abapgit_data_serializer DEFINITION PUBLIC SECTION. INTERFACES zif_abapgit_data_serializer . + PROTECTED SECTION. - METHODS dump_itab + PRIVATE SECTION. + + CONSTANTS c_max_records TYPE i VALUE 10000 ##NO_TEXT. + + METHODS convert_itab_to_json IMPORTING !ir_data TYPE REF TO data RETURNING @@ -20,22 +25,23 @@ CLASS zcl_abapgit_data_serializer DEFINITION !iv_name TYPE tadir-obj_name !it_where TYPE string_table RETURNING - VALUE(rr_data) TYPE REF TO data . - PRIVATE SECTION. + VALUE(rr_data) TYPE REF TO data + RAISING + zcx_abapgit_exception . ENDCLASS. -CLASS ZCL_ABAPGIT_DATA_SERIALIZER IMPLEMENTATION. +CLASS zcl_abapgit_data_serializer IMPLEMENTATION. - METHOD dump_itab. + METHOD convert_itab_to_json. DATA lo_ajson TYPE REF TO zcl_abapgit_ajson. DATA lv_string TYPE string. DATA lx_ajson TYPE REF TO zcx_abapgit_ajson_error. - FIELD-SYMBOLS TYPE ANY TABLE. + FIELD-SYMBOLS TYPE ANY TABLE. ASSIGN ir_data->* TO . @@ -57,8 +63,10 @@ CLASS ZCL_ABAPGIT_DATA_SERIALIZER IMPLEMENTATION. METHOD read_database_table. + DATA lv_records TYPE i. DATA lv_where LIKE LINE OF it_where. - FIELD-SYMBOLS: TYPE ANY TABLE. + + FIELD-SYMBOLS TYPE ANY TABLE. rr_data = zcl_abapgit_data_utils=>build_table_itab( iv_name ). ASSIGN rr_data->* TO . @@ -70,6 +78,12 @@ CLASS ZCL_ABAPGIT_DATA_SERIALIZER IMPLEMENTATION. SELECT * FROM (iv_name) INTO TABLE . ENDIF. + lv_records = lines( ). + IF lv_records > c_max_records. + zcx_abapgit_exception=>raise( |Too many records selected from table { iv_name + } (selected { lv_records }, max { c_max_records })| ). + ENDIF. + ENDMETHOD. @@ -80,20 +94,19 @@ CLASS ZCL_ABAPGIT_DATA_SERIALIZER IMPLEMENTATION. DATA ls_file LIKE LINE OF rt_files. DATA lr_data TYPE REF TO data. - ls_file-path = zif_abapgit_data_config=>c_default_path. lt_configs = ii_config->get_configs( ). LOOP AT lt_configs INTO ls_config. ASSERT ls_config-type = zif_abapgit_data_config=>c_data_type-tabu. " todo - ASSERT NOT ls_config-name IS INITIAL. + ASSERT ls_config-name IS NOT INITIAL. lr_data = read_database_table( iv_name = ls_config-name it_where = ls_config-where ). ls_file-filename = zcl_abapgit_data_utils=>build_filename( ls_config ). - ls_file-data = dump_itab( lr_data ). + ls_file-data = convert_itab_to_json( lr_data ). ls_file-sha1 = zcl_abapgit_hash=>sha1_blob( ls_file-data ). APPEND ls_file TO rt_files. diff --git a/src/data/zcl_abapgit_data_utils.clas.abap b/src/data/zcl_abapgit_data_utils.clas.abap index 77defd8e3..b11550c4a 100644 --- a/src/data/zcl_abapgit_data_utils.clas.abap +++ b/src/data/zcl_abapgit_data_utils.clas.abap @@ -3,30 +3,31 @@ CLASS zcl_abapgit_data_utils DEFINITION CREATE PUBLIC . PUBLIC SECTION. + CLASS-METHODS build_table_itab IMPORTING !iv_name TYPE tadir-obj_name RETURNING VALUE(rr_data) TYPE REF TO data . - CLASS-METHODS build_filename IMPORTING - is_config TYPE zif_abapgit_data_config=>ty_config + !is_config TYPE zif_abapgit_data_config=>ty_config RETURNING - VALUE(rv_filename) TYPE string. - + VALUE(rv_filename) TYPE string . PROTECTED SECTION. PRIVATE SECTION. ENDCLASS. -CLASS ZCL_ABAPGIT_DATA_UTILS IMPLEMENTATION. +CLASS zcl_abapgit_data_utils IMPLEMENTATION. METHOD build_filename. - rv_filename = to_lower( |{ is_config-name }.{ is_config-type }.json| ). + rv_filename = to_lower( |{ is_config-name }.{ is_config-type }.{ zif_abapgit_data_config=>c_default_format }| ). + + REPLACE ALL OCCURRENCES OF '/' IN rv_filename WITH '#'. ENDMETHOD. diff --git a/src/data/zcl_abapgit_data_utils.clas.testclasses.abap b/src/data/zcl_abapgit_data_utils.clas.testclasses.abap new file mode 100644 index 000000000..10cecbe31 --- /dev/null +++ b/src/data/zcl_abapgit_data_utils.clas.testclasses.abap @@ -0,0 +1,34 @@ +CLASS ltcl_data_utils_test DEFINITION FINAL + FOR TESTING + RISK LEVEL HARMLESS + DURATION SHORT. + + PRIVATE SECTION. + + METHODS build_filename FOR TESTING. + +ENDCLASS. + +CLASS ltcl_data_utils_test IMPLEMENTATION. + + METHOD build_filename. + + DATA ls_config TYPE zif_abapgit_data_config=>ty_config. + + ls_config-name = 'T100'. + ls_config-type = 'TABU'. + + cl_abap_unit_assert=>assert_equals( + act = zcl_abapgit_data_utils=>build_filename( ls_config ) + exp = 't100.tabu.json' ). + + ls_config-name = '/NSPC/T200'. + ls_config-type = 'TABU'. + + cl_abap_unit_assert=>assert_equals( + act = zcl_abapgit_data_utils=>build_filename( ls_config ) + exp = '#nspc#t200.tabu.json' ). + + ENDMETHOD. + +ENDCLASS. diff --git a/src/data/zcl_abapgit_data_utils.clas.xml b/src/data/zcl_abapgit_data_utils.clas.xml index 46a8a851e..091a2b8a0 100644 --- a/src/data/zcl_abapgit_data_utils.clas.xml +++ b/src/data/zcl_abapgit_data_utils.clas.xml @@ -10,6 +10,7 @@ X X X + X diff --git a/src/data/zif_abapgit_data_config.intf.abap b/src/data/zif_abapgit_data_config.intf.abap index 3792fdbc4..1a0a9d15d 100644 --- a/src/data/zif_abapgit_data_config.intf.abap +++ b/src/data/zif_abapgit_data_config.intf.abap @@ -14,6 +14,8 @@ INTERFACE zif_abapgit_data_config ty_config_tt TYPE SORTED TABLE OF ty_config WITH UNIQUE KEY type name . CONSTANTS c_default_path TYPE string VALUE '/data/' ##NO_TEXT. + CONSTANTS c_default_format TYPE string VALUE 'json' ##NO_TEXT. + CONSTANTS c_config TYPE string VALUE 'conf' ##NO_TEXT. CONSTANTS: BEGIN OF c_data_type, tabu TYPE ty_data_type VALUE 'TABU', diff --git a/src/data/zif_abapgit_data_config.intf.xml b/src/data/zif_abapgit_data_config.intf.xml index 12fc45be9..b88cd743c 100644 --- a/src/data/zif_abapgit_data_config.intf.xml +++ b/src/data/zif_abapgit_data_config.intf.xml @@ -5,7 +5,7 @@ ZIF_ABAPGIT_DATA_CONFIG E - abapGit - Data config + abapGit - Data Config 2 1 X diff --git a/src/data/zif_abapgit_data_deserializer.intf.abap b/src/data/zif_abapgit_data_deserializer.intf.abap index 17de4579d..a653489b7 100644 --- a/src/data/zif_abapgit_data_deserializer.intf.abap +++ b/src/data/zif_abapgit_data_deserializer.intf.abap @@ -1,25 +1,27 @@ INTERFACE zif_abapgit_data_deserializer PUBLIC . + TYPES: BEGIN OF ty_result, table TYPE tadir-obj_name, deletes TYPE REF TO data, updates TYPE REF TO data, inserts TYPE REF TO data, END OF ty_result. + TYPES: ty_results TYPE STANDARD TABLE OF ty_result WITH KEY table. METHODS deserialize IMPORTING !ii_config TYPE REF TO zif_abapgit_data_config !it_files TYPE zif_abapgit_definitions=>ty_files_tt RETURNING - VALUE(rs_result) TYPE ty_result + VALUE(rt_result) TYPE ty_results RAISING zcx_abapgit_exception . METHODS actualize IMPORTING - is_result TYPE ty_result + it_result TYPE ty_results RAISING zcx_abapgit_exception . diff --git a/src/data/zif_abapgit_data_deserializer.intf.xml b/src/data/zif_abapgit_data_deserializer.intf.xml index e91e71f1f..81f6e7c3c 100644 --- a/src/data/zif_abapgit_data_deserializer.intf.xml +++ b/src/data/zif_abapgit_data_deserializer.intf.xml @@ -5,7 +5,7 @@ ZIF_ABAPGIT_DATA_DESERIALIZER E - abapGit - Data deserializer + abapGit - Data Deserializer 2 1 X diff --git a/src/data/zif_abapgit_data_serializer.intf.xml b/src/data/zif_abapgit_data_serializer.intf.xml index e29e518f6..6d11b79cc 100644 --- a/src/data/zif_abapgit_data_serializer.intf.xml +++ b/src/data/zif_abapgit_data_serializer.intf.xml @@ -5,7 +5,7 @@ ZIF_ABAPGIT_DATA_SERIALIZER E - abapGit - Data serializer + abapGit - Data Serializer 2 1 X diff --git a/src/repo/zcl_abapgit_repo.clas.abap b/src/repo/zcl_abapgit_repo.clas.abap index c28b2544e..ecd6cafbd 100644 --- a/src/repo/zcl_abapgit_repo.clas.abap +++ b/src/repo/zcl_abapgit_repo.clas.abap @@ -217,7 +217,7 @@ ENDCLASS. -CLASS ZCL_ABAPGIT_REPO IMPLEMENTATION. +CLASS zcl_abapgit_repo IMPLEMENTATION. METHOD bind_listener. @@ -331,6 +331,7 @@ CLASS ZCL_ABAPGIT_REPO IMPLEMENTATION. METHOD deserialize. DATA: lt_updated_files TYPE zif_abapgit_definitions=>ty_file_signatures_tt, + lt_result TYPE zif_abapgit_data_deserializer=>ty_results, lx_error TYPE REF TO zcx_abapgit_exception. find_remote_dot_abapgit( ). @@ -351,6 +352,7 @@ CLASS ZCL_ABAPGIT_REPO IMPLEMENTATION. zcx_abapgit_exception=>raise( |No transport request was supplied| ). ENDIF. + " Deserialize objects TRY. lt_updated_files = zcl_abapgit_objects=>deserialize( io_repo = me @@ -364,9 +366,15 @@ CLASS ZCL_ABAPGIT_REPO IMPLEMENTATION. APPEND get_dot_abapgit( )->get_signature( ) TO lt_updated_files. + update_local_checksums( lt_updated_files ). + + " Deserialize data (no save to database, just test for now) + lt_result = zcl_abapgit_data_factory=>get_deserializer( )->deserialize( + ii_config = get_data_config( ) + it_files = get_files_remote( ) ). + CLEAR: mt_local. - update_local_checksums( lt_updated_files ). update_last_deserialize( ). reset_status( ).