Merge branch 'main' into hvam/unit0511

src/zcl_excel_reader_2007.clas.abap

@@ -274,6 +274,11 @@ CLASS zcl_excel_reader_2007 DEFINITION
       RETURNING
         VALUE(rp_result) TYPE string .
     METHODS resolve_referenced_formulae .
+    METHODS unescape_string_value
+      IMPORTING
+        i_value       TYPE string
+      RETURNING
+        VALUE(result) TYPE string.
     METHODS get_dxf_style_guid
       IMPORTING
         !io_ixml_dxf         TYPE REF TO if_ixml_element
@@ -963,7 +968,7 @@ CLASS zcl_excel_reader_2007 IMPLEMENTATION.
 *   §1.1 - "simple" strings
 *                Example:  see above
 *--------------------------------------------------------------------*
-          <ls_shared_string>-value = lo_node_si_child->get_value( ).
+          <ls_shared_string>-value = unescape_string_value( lo_node_si_child->get_value( ) ).
         ELSE.
 *--------------------------------------------------------------------*
 *   §1.2 - rich text formatted strings
@@ -985,7 +990,7 @@ CLASS zcl_excel_reader_2007 IMPLEMENTATION.
             " extract the <t>...</t> part of each <r>-tag
             lo_node_r_child_t ?= lo_node_si_child->find_from_name_ns( name = 't' uri = namespace-main ).
             IF lo_node_r_child_t IS BOUND.
-              lv_node_value = lo_node_r_child_t->get_value( ).
+              lv_node_value = unescape_string_value( lo_node_r_child_t->get_value( ) ).
               CONCATENATE <ls_shared_string>-value lv_node_value INTO <ls_shared_string>-value RESPECTING BLANKS.
               ls_rtf-length = strlen( lv_node_value ).
             ENDIF.
@@ -4101,6 +4106,42 @@ CLASS zcl_excel_reader_2007 IMPLEMENTATION.
   ENDMETHOD.
 
 
+  METHOD unescape_string_value.
+
+    DATA: lt_character_positions       TYPE TABLE OF i,
+          lv_character_position        TYPE i,
+          lv_character_position_plus_2 TYPE i,
+          lv_character_position_plus_6 TYPE i,
+          lv_unescaped_value           TYPE string.
+
+    " The text "_x...._", with "_x" not "_X", with exactly 4 ".", each being 0-9 a-f or A-F (case insensitive), is interpreted
+    " like Unicode character U+.... (e.g. "_x0041_" is rendered like "A") is for characters.
+    " To not interpret it, Excel replaces the first "_" with "_x005f_".
+    result = i_value.
+    IF result CS '_x'.
+      CLEAR lt_character_positions.
+      APPEND sy-fdpos TO lt_character_positions.
+      lv_character_position = sy-fdpos + 1.
+      WHILE result+lv_character_position CS '_x'.
+        ADD sy-fdpos TO lv_character_position.
+        APPEND lv_character_position TO lt_character_positions.
+        ADD 1 TO lv_character_position.
+      ENDWHILE.
+      SORT lt_character_positions BY table_line DESCENDING.
+      LOOP AT lt_character_positions INTO lv_character_position.
+        lv_character_position_plus_2 = lv_character_position + 2.
+        lv_character_position_plus_6 = lv_character_position + 6.
+        IF substring( val = result off = lv_character_position_plus_2 len = 4 ) CO '0123456789ABCDEFGHIJKLMNOPQRSTUVWabcdefghijklmnopqrstuvw'
+          AND substring( val = result off = lv_character_position_plus_6 len = 1 ) = '_'.
+          lv_unescaped_value = cl_abap_conv_in_ce=>uccp( to_upper( substring( val = result off = lv_character_position_plus_2 len = 4 ) ) ).
+          REPLACE SECTION OFFSET lv_character_position LENGTH 7 OF result WITH lv_unescaped_value.
+        ENDIF.
+      ENDLOOP.
+    ENDIF.
+
+  ENDMETHOD.
+
+
   METHOD zif_excel_reader~load.
 *--------------------------------------------------------------------*
 * ToDos:

src/zcl_excel_reader_2007.clas.testclasses.abap

@@ -0,0 +1,55 @@
+*"* use this source file for your ABAP unit test classes
+
+CLASS ltc_unescape_string_value DEFINITION DEFERRED.
+CLASS zcl_excel_reader_2007 DEFINITION LOCAL FRIENDS
+    ltc_unescape_string_value.
+
+
+CLASS ltc_unescape_string_value DEFINITION
+      FOR TESTING
+      DURATION SHORT
+      RISK LEVEL HARMLESS.
+  PRIVATE SECTION.
+
+    METHODS escaped_character_inside_text FOR TESTING.
+    METHODS no_escaping FOR TESTING.
+    METHODS one_escaped_character FOR TESTING.
+    METHODS two_escaped_characters FOR TESTING.
+
+    METHODS run_cut
+      IMPORTING
+        input TYPE string
+        exp   TYPE string.
+
+ENDCLASS.
+
+
+CLASS ltc_unescape_string_value IMPLEMENTATION.
+
+  METHOD escaped_character_inside_text.
+    run_cut( input = 'start _x0000_ end' exp = |start { cl_abap_conv_in_ce=>uccp( '0000' ) } end| ).
+  ENDMETHOD.
+
+  METHOD no_escaping.
+    run_cut( input = 'no escaping' exp = 'no escaping' ).
+  ENDMETHOD.
+
+  METHOD one_escaped_character.
+    run_cut( input = '_x0000_' exp = cl_abap_conv_in_ce=>uccp( '0000' ) ).
+  ENDMETHOD.
+
+  METHOD run_cut.
+
+    DATA: lo_excel TYPE REF TO zcl_excel_reader_2007.
+
+    CREATE OBJECT lo_excel.
+
+    cl_abap_unit_assert=>assert_equals( act = lo_excel->unescape_string_value( input ) exp = exp msg = |input: { input }| ).
+
+  ENDMETHOD.
+
+  METHOD two_escaped_characters.
+    run_cut( input = '_x0000_ and _xFFFF_' exp = |{ cl_abap_conv_in_ce=>uccp( '0000' ) } and { cl_abap_conv_in_ce=>uccp( 'FFFF' ) }| ).
+  ENDMETHOD.
+
+ENDCLASS.

src/zcl_excel_reader_2007.clas.xml

@@ -10,6 +10,7 @@
     <CLSCCINCL>X</CLSCCINCL>
     <FIXPT>X</FIXPT>
     <UNICODE>X</UNICODE>
+    <WITH_UNIT_TESTS>X</WITH_UNIT_TESTS>
    </VSEOCLASS>
    <TPOOL>
     <item>

src/zcl_excel_reader_huge_file.clas.abap

@@ -206,7 +206,7 @@ CLASS zcl_excel_reader_huge_file IMPLEMENTATION.
 
     lt_shared_strings = read_shared_strings( lo_reader ).
     LOOP AT lt_shared_strings ASSIGNING <lv_shared_string>.
-      ls_shared_string-value = <lv_shared_string>.
+      ls_shared_string-value = unescape_string_value( <lv_shared_string> ).
       APPEND ls_shared_string TO shared_strings.
     ENDLOOP.
 

src/zcl_excel_theme_fmt_scheme.clas.abap

@@ -19,15 +19,18 @@ CLASS zcl_excel_theme_fmt_scheme DEFINITION
     METHODS get_default_fmt
       RETURNING
         VALUE(rv_string) TYPE string .
+
+    METHODS parse_string
+      IMPORTING iv_string      TYPE string
+      RETURNING VALUE(ri_node) TYPE REF TO if_ixml_node.
 ENDCLASS.
 
 
 
-CLASS zcl_excel_theme_fmt_scheme IMPLEMENTATION.
+CLASS ZCL_EXCEL_THEME_FMT_SCHEME IMPLEMENTATION.
 
 
   METHOD build_xml.
-    DATA: lo_xml TYPE REF TO cl_xml_document.
     DATA: lo_node TYPE REF TO if_ixml_node.
     DATA: lo_elements TYPE REF TO if_ixml_element.
     CHECK io_document IS BOUND.
@@ -35,9 +38,7 @@ CLASS zcl_excel_theme_fmt_scheme IMPLEMENTATION.
     IF lo_elements IS BOUND.
 
       IF fmt_scheme IS INITIAL.
-        CREATE OBJECT lo_xml.
+        lo_node = parse_string( get_default_fmt( ) ).
-        lo_xml->parse_string( get_default_fmt( ) ).
-        lo_node = lo_xml->get_first_node( ).
         lo_elements->append_child( new_child = lo_node ).
       ELSE.
         lo_elements->append_child( new_child = fmt_scheme ).
@@ -191,4 +192,28 @@ CLASS zcl_excel_theme_fmt_scheme IMPLEMENTATION.
   METHOD load.
     fmt_scheme = zcl_excel_common=>clone_ixml_with_namespaces( io_fmt_scheme ).
   ENDMETHOD.                    "load
+
+
+  METHOD parse_string.
+    DATA li_stream   TYPE REF TO if_ixml_istream.
+    DATA li_ixml     TYPE REF TO if_ixml.
+    DATA li_document TYPE REF TO if_ixml_document.
+    DATA li_factory  TYPE REF TO if_ixml_stream_factory.
+    DATA li_parser   TYPE REF TO if_ixml_parser.
+    DATA li_istream  TYPE REF TO if_ixml_istream.
+
+    li_ixml = cl_ixml=>create( ).
+    li_document = li_ixml->create_document( ).
+    li_factory = li_ixml->create_stream_factory( ).
+    li_istream = li_factory->create_istream_string( iv_string ).
+    li_parser = li_ixml->create_parser(
+      stream_factory = li_factory
+      istream        = li_istream
+      document       = li_document ).
+    li_parser->add_strip_space_element( ).
+    li_parser->parse( ).
+    li_istream->close( ).
+    ri_node = li_document->get_first_child( ).
+
+  ENDMETHOD.
 ENDCLASS.

src/zcl_excel_theme_fmt_scheme.clas.testclasses.abap

@@ -0,0 +1,54 @@
+CLASS ltcl_test DEFINITION FOR TESTING DURATION SHORT RISK LEVEL HARMLESS.
+  PRIVATE SECTION.
+    METHODS build_xml FOR TESTING.
+
+    DATA mi_ixml TYPE REF TO if_ixml.
+    DATA mi_document TYPE REF TO if_ixml_document.
+    METHODS setup.
+    METHODS render
+      RETURNING
+        VALUE(rv_xml) TYPE string.
+ENDCLASS.
+
+
+CLASS ltcl_test IMPLEMENTATION.
+
+  METHOD setup.
+    mi_ixml = cl_ixml=>create( ).
+    mi_document = mi_ixml->create_document( ).
+  ENDMETHOD.
+
+  METHOD render.
+    DATA li_ostream  TYPE REF TO if_ixml_ostream.
+    DATA li_renderer TYPE REF TO if_ixml_renderer.
+    DATA li_factory  TYPE REF TO if_ixml_stream_factory.
+
+    li_factory = mi_ixml->create_stream_factory( ).
+    li_ostream = li_factory->create_ostream_cstring( rv_xml ).
+    li_renderer = mi_ixml->create_renderer(
+      ostream  = li_ostream
+      document = mi_document ).
+    li_renderer->render( ).
+  ENDMETHOD.
+
+  METHOD build_xml.
+    DATA lo_theme_fmt TYPE REF TO zcl_excel_theme_fmt_scheme.
+    DATA li_ixml      TYPE REF TO if_ixml.
+    DATA li_document  TYPE REF TO if_ixml_document.
+    DATA lv_xml       TYPE string.
+
+    mi_document->create_simple_element(
+      name   = zcl_excel_theme=>c_theme_elements
+      parent = mi_document ).
+
+    CREATE OBJECT lo_theme_fmt.
+    lo_theme_fmt->build_xml( mi_document ).
+
+    lv_xml = render( ).
+
+    cl_abap_unit_assert=>assert_char_cp(
+      act = lv_xml
+      exp = '*<a:fmtScheme name="Office">*' ).
+  ENDMETHOD.
+
+ENDCLASS.

src/zcl_excel_theme_fmt_scheme.clas.xml

@@ -10,6 +10,7 @@
     <CLSCCINCL>X</CLSCCINCL>
     <FIXPT>X</FIXPT>
     <UNICODE>X</UNICODE>
+    <WITH_UNIT_TESTS>X</WITH_UNIT_TESTS>
    </VSEOCLASS>
   </asx:values>
  </asx:abap>

src/zcl_excel_writer_2007.clas.abap

@@ -42,6 +42,7 @@ CLASS zcl_excel_writer_2007 DEFINITION
     CONSTANTS cl_xl_drawing_for_comments TYPE string VALUE 'xl/drawings/vmlDrawing#.vml'. "#EC NOTEXT
     CONSTANTS c_xl_drawings_vml_rels TYPE string VALUE 'xl/drawings/_rels/vmlDrawing#.vml.rels'. "#EC NOTEXT
     DATA ixml TYPE REF TO if_ixml.
+    DATA control_characters TYPE string.
 
     METHODS create_xl_sheet_sheet_data
       IMPORTING
@@ -193,6 +194,11 @@ CLASS zcl_excel_writer_2007 DEFINITION
     METHODS create_xl_drawings_vml_rels
       RETURNING
         VALUE(ep_content) TYPE xstring .
+    METHODS escape_string_value
+      IMPORTING
+        !iv_value     TYPE zexcel_cell_value
+      RETURNING
+        VALUE(result) TYPE zexcel_cell_value.
     METHODS set_vml_shape_footer
       IMPORTING
         !is_footer        TYPE zexcel_s_worksheet_head_foot
@@ -289,7 +295,21 @@ CLASS zcl_excel_writer_2007 IMPLEMENTATION.
 
 
   METHOD constructor.
+    DATA: lt_unicode_point_codes TYPE TABLE OF string,
+          lv_unicode_point_code  TYPE i.
+
     me->ixml = cl_ixml=>create( ).
+
+    SPLIT '0,1,2,3,4,5,6,7,8,' " U+0000 to U+0008
+       && '11,12,'             " U+000B, U+000C
+       && '14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,' " U+000E to U+001F
+       && '65534,65535'        " U+FFFE, U+FFFF
+      AT ',' INTO TABLE lt_unicode_point_codes.
+    control_characters = ``.
+    LOOP AT lt_unicode_point_codes INTO lv_unicode_point_code.
+      control_characters = control_characters && cl_abap_conv_in_ce=>uccpi( lv_unicode_point_code ).
+    ENDLOOP.
+
   ENDMETHOD.
 
 
@@ -3633,7 +3653,8 @@ CLASS zcl_excel_writer_2007 IMPLEMENTATION.
               OR boolc( contains( val = <fs_sheet_string>-string_value end = ` ` ) ) = abap_true.
           lo_sub_element->set_attribute( name = 'space' namespace = 'xml' value = 'preserve' ).
         ENDIF.
-        lo_sub_element->set_value( value = <fs_sheet_string>-string_value ).
+        lv_value = escape_string_value( <fs_sheet_string>-string_value ).
+        lo_sub_element->set_value( value = lv_value ).
       ELSE.
         LOOP AT <fs_sheet_string>-rtf_tab ASSIGNING <fs_rtf>.
           lo_sub_element = lo_document->create_simple_element( name   = lc_xml_node_r
@@ -3645,6 +3666,7 @@ CLASS zcl_excel_writer_2007 IMPLEMENTATION.
             CATCH cx_sy_range_out_of_bounds.
               EXIT.
           ENDTRY.
+          lv_value = escape_string_value( lv_value ).
           IF <fs_rtf>-font IS NOT INITIAL.
             lo_font_element = lo_document->create_simple_element( name   = lc_xml_node_rpr
                                                                   parent = lo_sub_element ).
@@ -6111,6 +6133,34 @@ CLASS zcl_excel_writer_2007 IMPLEMENTATION.
   ENDMETHOD.
 
 
+  METHOD escape_string_value.
+
+    DATA: lt_character_positions TYPE TABLE OF i,
+          lv_character_position  TYPE i,
+          lv_escaped_value       TYPE string.
+
+    result = iv_value.
+    IF result CA control_characters.
+
+      CLEAR lt_character_positions.
+      APPEND sy-fdpos TO lt_character_positions.
+      lv_character_position = sy-fdpos + 1.
+      WHILE result+lv_character_position CA control_characters.
+        ADD sy-fdpos TO lv_character_position.
+        APPEND lv_character_position TO lt_character_positions.
+        ADD 1 TO lv_character_position.
+      ENDWHILE.
+      SORT lt_character_positions BY table_line DESCENDING.
+
+      LOOP AT lt_character_positions INTO lv_character_position.
+        lv_escaped_value = |_x{ cl_abap_conv_out_ce=>uccp( substring( val = result off = lv_character_position len = 1 ) ) }_|.
+        REPLACE SECTION OFFSET lv_character_position LENGTH 1 OF result WITH lv_escaped_value.
+      ENDLOOP.
+    ENDIF.
+
+  ENDMETHOD.
+
+
   METHOD flag2bool.
 
 

src/zcl_excel_writer_2007.clas.testclasses.abap

@@ -2,9 +2,11 @@
 
 CLASS ltc_column_formula DEFINITION DEFERRED.
 CLASS ltc_is_formula_shareable DEFINITION DEFERRED.
+CLASS ltc_escape_string_value DEFINITION DEFERRED.
 CLASS zcl_excel_writer_2007 DEFINITION LOCAL FRIENDS
     ltc_column_formula
-    ltc_is_formula_shareable.
+    ltc_is_formula_shareable
+    ltc_escape_string_value.
 
 CLASS ltc_column_formula DEFINITION FOR TESTING
     RISK LEVEL HARMLESS
@@ -47,6 +49,26 @@ CLASS ltc_is_formula_shareable DEFINITION FOR TESTING
 ENDCLASS.
 
 
+CLASS ltc_escape_string_value DEFINITION
+      FOR TESTING
+      DURATION SHORT
+      RISK LEVEL HARMLESS.
+
+  PRIVATE SECTION.
+
+    METHODS escaped_character_inside_text FOR TESTING.
+    METHODS no_escaping FOR TESTING.
+    METHODS one_escaped_character FOR TESTING.
+    METHODS two_escaped_characters FOR TESTING.
+
+    METHODS run_cut
+      IMPORTING
+        input TYPE string
+        exp   TYPE string.
+
+ENDCLASS.
+
+
 CLASS ltc_column_formula IMPLEMENTATION.
 
   METHOD one_column_formula.
@@ -168,3 +190,51 @@ CLASS ltc_is_formula_shareable IMPLEMENTATION.
   ENDMETHOD.
 
 ENDCLASS.
+
+
+CLASS ltc_escape_string_value IMPLEMENTATION.
+
+  METHOD escaped_character_inside_text.
+    run_cut( input = 'start U+0000 end' exp = 'start _x0000_ end' ).
+  ENDMETHOD.
+
+  METHOD no_escaping.
+    run_cut( input = 'no escaping' exp = 'no escaping' ).
+  ENDMETHOD.
+
+  METHOD one_escaped_character.
+    run_cut( input = 'U+0000' exp = '_x0000_' ).
+  ENDMETHOD.
+
+  METHOD run_cut.
+
+    DATA: lo_excel    TYPE REF TO zcl_excel_writer_2007,
+          lv_input    TYPE string,
+          lt_matches  TYPE match_result_tab,
+          ls_match    TYPE match_result,
+          ls_submatch TYPE submatch_result,
+          lv_hexa     TYPE c LENGTH 4,
+          lv_uccp     TYPE c LENGTH 1.
+
+    CREATE OBJECT lo_excel.
+
+    lv_input = input.
+    FIND ALL OCCURRENCES OF REGEX 'U\+(....)' IN lv_input RESULTS lt_matches.
+    SORT lt_matches BY offset DESCENDING.
+    LOOP AT lt_matches INTO ls_match.
+      READ TABLE ls_match-submatches INDEX 1 INTO ls_submatch.
+      ASSERT sy-subrc = 0.
+      lv_hexa = substring( val = lv_input off = ls_submatch-offset len = ls_submatch-length ).
+      lv_uccp = cl_abap_conv_in_ce=>uccp( lv_hexa ).
+      REPLACE SECTION OFFSET ls_match-offset LENGTH ls_match-length OF lv_input WITH lv_uccp.
+    ENDLOOP.
+
+    cl_abap_unit_assert=>assert_equals( act = lo_excel->escape_string_value( lv_input ) exp = exp msg = |input: { input }| ).
+
+  ENDMETHOD.
+
+  METHOD two_escaped_characters.
+    run_cut( input = 'U+0000 and U+FFFF' exp = '_x0000_ and _xFFFF_' ).
+  ENDMETHOD.
+
+ENDCLASS.

src/zcl_excel_writer_huge_file.clas.abap

@@ -50,32 +50,20 @@ CLASS zcl_excel_writer_huge_file IMPLEMENTATION.
         unique_count TYPE string,
       END OF ts_root.
 
-    DATA:
-      lv_last_allowed_char TYPE c,
-      lv_invalid           TYPE string.
-
     DATA:
       lo_iterator  TYPE REF TO zcl_excel_collection_iterator,
       lo_worksheet TYPE REF TO zcl_excel_worksheet.
 
     DATA:
-      ls_root          TYPE ts_root,
+      ls_root           TYPE ts_root,
-      lt_cell_data     TYPE zexcel_t_cell_data_unsorted,
+      lt_cell_data      TYPE zexcel_t_cell_data_unsorted,
-      ls_shared_string TYPE zexcel_s_shared_string,
+      ls_shared_string  TYPE zexcel_s_shared_string,
-      lv_sytabix       TYPE i.
+      lv_sytabix        TYPE i,
+      lt_shared_strings TYPE TABLE OF zexcel_s_shared_string.
 
     FIELD-SYMBOLS:
       <sheet_content>     TYPE zexcel_s_cell_data.
 
-**********************************************************************
-* STEP 0: Build Regex for invalid characters
-    " uccpi returns 2 chars but for this specific input 1 char is enough
-    CASE cl_abap_char_utilities=>charsize.
-      WHEN 1.lv_last_allowed_char = cl_abap_conv_in_ce=>uccpi( 255 ).  " FF     in non-Unicode
-      WHEN 2.lv_last_allowed_char = cl_abap_conv_in_ce=>uccpi( 65533 )." FFFD   in Unicode
-    ENDCASE.
-    CONCATENATE '[^\n\t\r -' lv_last_allowed_char ']' INTO lv_invalid.
-
 **********************************************************************
 * STEP 1: Collect strings from each worksheet
 
@@ -103,8 +91,6 @@ CLASS zcl_excel_writer_huge_file IMPLEMENTATION.
       lv_sytabix = sy-tabix - 1.
       ls_shared_string-string_no = lv_sytabix.
       ls_shared_string-string_value = <sheet_content>-cell_value.
-      REPLACE ALL OCCURRENCES OF REGEX lv_invalid
-           IN ls_shared_string-string_value WITH ` `.
       INSERT ls_shared_string INTO TABLE shared_strings.
 
     ENDLOOP.
@@ -112,9 +98,16 @@ CLASS zcl_excel_writer_huge_file IMPLEMENTATION.
 **********************************************************************
 * STEP 2: Create XML
 
+    " Escape the string values, use of standard table in order to keep same sort order as in sorted table SHARED_STRINGS.
+    CLEAR lt_shared_strings.
+    LOOP AT shared_strings INTO ls_shared_string.
+      ls_shared_string-string_value = escape_string_value( ls_shared_string-string_value ).
+      APPEND ls_shared_string TO lt_shared_strings.
+    ENDLOOP.
+
     CALL TRANSFORMATION zexcel_tr_shared_strings
       SOURCE root = ls_root
-             shared_strings = shared_strings
+             shared_strings = lt_shared_strings
       OPTIONS xml_header = 'full'
       RESULT XML ep_content.
 
@@ -807,8 +800,7 @@ CLASS zcl_excel_writer_huge_file IMPLEMENTATION.
       <cell>-formula = <content>-cell_formula.
       <cell>-type    = <content>-data_type.
       IF <cell>-type = 's'.
-        <cell>-value = me->get_shared_string_index( ip_cell_value = <content>-cell_value
+        <cell>-value = me->get_shared_string_index( ip_cell_value = <content>-cell_value ).
-                                                    it_rtf        = <content>-rtf_tab ).
       ELSE.
         <cell>-value = <content>-cell_value.
       ENDIF.