Fix the bug regarding exceeding column 72 with UTF-8 Japanese literal (#621)

Fixed the bug regarding exceeding column 72 with UTF-8 Japanese literal.

detail:

In the case of UTF-8 Japanese strings are used, '\n' would be inserted to column 73.
Due to the difference in the number of bytes between SJIS and UTF-8, '\n' may be inserted in a different position than in SJIS, resulting in an error.

Author: kio-watanabe

cobj/cobj.c

@@ -705,6 +705,27 @@ int utf8_hankaku_kana(const unsigned char *p) {
   }
   return 0;
 }
+
+int utf8_calc_sjis_column(const unsigned char *p, int column) {
+  const unsigned char *start = p;
+  int char_size = 0;
+  int i = 0;
+
+  while (i < column && *p != '\0') {
+    char_size = COB_U8BYTE_1(*p);
+    if (char_size == 1) {
+      i++;
+      p++;
+    } else if (char_size == 3 && utf8_hankaku_kana(p)) {
+      i++;
+      p += char_size;
+    } else {
+      i += 2;
+      p += char_size;
+    }
+  }
+  return p - start;
+}
 #endif /*I18N_UTF8*/
 
 /*

cobj/cobj.h

@@ -200,6 +200,7 @@ extern void utf8_spc_to_ascii(char *);
 extern int utf8_national_length(const unsigned char *str, int len);
 extern size_t utf8_calc_sjis_size(const unsigned char *data, int len);
 extern int utf8_hankaku_kana(const unsigned char *data);
+extern int utf8_calc_sjis_column(const unsigned char *p, int column);
 #else  /*!I18N_UTF8*/
 extern const unsigned char *sjis_pick(const unsigned char *);
 extern size_t sjis_strlen(const unsigned char *);

cobj/pplex.l.m4

@@ -1179,7 +1179,12 @@ start:
 		}
 
 		/* remove it */
+#ifdef I18N_UTF8
+        p = (unsigned char *) buff;
+		strcpy (buff + utf8_calc_sjis_column(p, cb_text_column), "\n");
+#else /*!I18N_UTF8*/
 		strcpy (buff + cb_text_column, "\n");
+#endif /*I18N_UTF8*/
 		last_line_2 = cb_source_line;
 		n = cb_text_column + 1;
 	}

tests/cobol_utf8.src/pic-x.at

@@ -614,6 +614,27 @@ AT_CHECK([COB_TERMINAL_ENCODING=UTF-8 java prog], [0],
 
 AT_CLEANUP
 
+AT_SETUP([Exceed column 72 with Japanese literal])
+export LC_ALL=''
+
+AT_DATA([prog.cob], [       
+       IDENTIFICATION   DIVISION.
+       PROGRAM-ID.      prog.
+       DATA DIVISION.
+       WORKING-STORAGE SECTION.
+       01 str PIC X(5) VALUE                                     'ｱｲｳｴｵ'.
+       PROCEDURE DIVISION.
+       DISPLAY str.
+])
+
+AT_CHECK([cobj prog.cob])
+AT_CHECK([java prog > out1.txt])
+AT_CHECK([echo -n 'ｱｲｳｴｵ
+' | nkf -x --ic=UTF-8 --oc=Shift_JIS > out2.txt])
+AT_CHECK([diff out1.txt out2.txt])
+
+AT_CLEANUP
+
 #AT_SETUP([Readable string literals])
 #export LC_ALL=''
 ## Older compilers converts string literals "日本語" in COBOL source code