Implement string tokenization functionalities as defined in Fortran 2023

src/core_utils/string_core_utils.F90

@@ -6,9 +6,16 @@ module string_core_utils
     public :: core_to_str                ! Convert integer to left justified string
     public :: core_int_date_to_yyyymmdd  ! Convert encoded date integer to "yyyy-mm-dd" format
     public :: core_int_seconds_to_hhmmss ! Convert integer seconds past midnight to "hh:mm:ss" format
-    public :: core_stringify             ! Convert one or more values of any intrinsic data types to a character string for pretty printing
+    public :: split                      ! Parse a string into tokens, one at a time
+    public :: stringify                  ! Convert one or more values of any intrinsic data types to a character string for pretty printing
+    public :: tokenize                   ! Parse a string into tokens
 
-CONTAINS
+    interface tokenize
+        module procedure tokenize_into_first_last
+        module procedure tokenize_into_tokens_separator
+    end interface tokenize
+
+contains
 
     character(len=10) pure function core_to_str(n)
         ! return default integer as a left justified string
@@ -59,18 +66,59 @@ character(len=8) pure function core_int_seconds_to_hhmmss (seconds)
 
     end function core_int_seconds_to_hhmmss
 
+    !> Parse a string into tokens, one at a time. Each character in `set` is a token delimiter.
+    !> If `back` is absent or is present with the value `.false.`, `pos` is assigned the position of the leftmost
+    !> token delimiter in `string` whose position is greater than `pos`, or if there is no such character, it
+    !> is assigned a value one greater than the length of `string`. This identifies a token with starting
+    !> position one greater than the value of `pos` on invocation, and ending position one less than the
+    !> value of `pos` on return.
+    !> If `back` is present with the value `.true.`, `pos` is assigned the position of the rightmost token delimiter
+    !> in `string` whose position is less than `pos`, or if there is no such character, it is assigned the value
+    !> zero. This identifies a token with ending position one less than the value of `pos` on invocation, and
+    !> starting position one greater than the value of `pos` on return.
+    !> This subroutine implements the `split` intrinsic procedure as defined in the Fortran 2023 language standard
+    !> (Section 16.9.196). We implement it ourselves because the compiler support may take years to become widespread.
+    !> (KCW, 2025-10-29)
+    pure subroutine split(string, set, pos, back)
+        character(*), intent(in) :: string, set
+        integer, intent(inout) :: pos
+        logical, optional, intent(in) :: back
+
+        integer :: offset
+
+        if (present(back)) then
+            if (back) then
+                offset = max(min(pos, len(string) + 1), 1)
+                pos = scan(string(1:offset - 1), set, back=.true.)
+
+                return
+            end if
+        end if
+
+        offset = max(min(pos, len(string)), 0)
+        pos = scan(string(offset + 1:), set)
+
+        if (pos == 0) then
+            pos = len(string) + 1
+
+            return
+        end if
+
+        pos = offset + pos
+    end subroutine split
+
     !> Convert one or more values of any intrinsic data types to a character string for pretty printing.
     !> If `value` contains more than one element, the elements will be stringified, delimited by `separator`, then concatenated.
     !> If `value` contains exactly one element, the element will be stringified without using `separator`.
     !> If `value` contains zero element or is of unsupported data types, an empty character string is produced.
     !> If `separator` is not supplied, it defaults to ", " (i.e., a comma and a space).
     !> (KCW, 2024-02-04)
-    pure function core_stringify(value, separator)
+    pure function stringify(value, separator)
         use, intrinsic :: iso_fortran_env, only: int32, int64, real32, real64
 
         class(*), intent(in) :: value(:)
         character(*), optional, intent(in) :: separator
-        character(:), allocatable :: core_stringify
+        character(:), allocatable :: stringify
 
         integer, parameter :: sizelimit = 1024
 
@@ -87,7 +135,7 @@ pure function core_stringify(value, separator)
         n = min(size(value), sizelimit)
 
         if (n == 0) then
-            core_stringify = ''
+            stringify = ''
 
             return
         end if
@@ -162,12 +210,81 @@ pure function core_stringify(value, separator)
 
                 write(buffer, format) value
             class default
-                core_stringify = ''
+                stringify = ''
 
                 return
         end select
 
-        core_stringify = trim(buffer)
-    end function core_stringify
+        stringify = trim(buffer)
+    end function stringify
+
+    !> Parse a string into tokens. Each character in `set` is a token delimiter.
+    !> `first` is allocated with the lower bound equal to one and the upper bound equal to the number of tokens in `string`.
+    !> Each element is assigned, in array element order, the starting position of each token in `string`, in the order found.
+    !> `last` is allocated with the lower bound equal to one and the upper bound equal to the number of tokens in `string`.
+    !> Each element is assigned, in array element order, the ending position of each token in `string`, in the order found.
+    !> This subroutine implements the `tokenize` intrinsic procedure as defined in the Fortran 2023 language standard
+    !> (Section 16.9.210). We implement it ourselves because the compiler support may take years to become widespread.
+    !> (KCW, 2025-10-29)
+    pure subroutine tokenize_into_first_last(string, set, first, last)
+        character(*), intent(in) :: string, set
+        integer, allocatable, intent(out) :: first(:), last(:)
+
+        integer :: pos_start(len(string) + 1), pos_end(len(string) + 1)
+        integer :: l, n, pos
+
+        l = len(string)
+        n = 0
+        pos = 0
+
+        do while (pos < l + 1)
+            n = n + 1
+            pos_start(n) = pos + 1
+
+            call split(string, set, pos)
+
+            pos_end(n) = pos - 1
+        end do
+
+        allocate(first(n), last(n))
+
+        first(:) = pos_start(1:n)
+        last(:) = pos_end(1:n)
+    end subroutine tokenize_into_first_last
+
+    !> Parse a string into tokens. Each character in `set` is a token delimiter.
+    !> `tokens` is allocated with the lower bound equal to one and the upper bound equal to the number of tokens in `string`,
+    !> and with character length equal to the length of the longest token. It contains the tokens in `string`.
+    !> `separator` is allocated with the lower bound equal to one and the upper bound equal to one less than the number of
+    !> tokens in `string`, and with character length equal to one. It contains the token delimiters in `string`.
+    !> This subroutine implements the `tokenize` intrinsic procedure as defined in the Fortran 2023 language standard
+    !> (Section 16.9.210). We implement it ourselves because the compiler support may take years to become widespread.
+    !> (KCW, 2025-10-29)
+    pure subroutine tokenize_into_tokens_separator(string, set, tokens, separator)
+        character(*), intent(in) :: string, set
+        character(:), allocatable, intent(out) :: tokens(:)
+        character(:), allocatable, optional, intent(out) :: separator(:)
+
+        integer, allocatable :: first(:), last(:)
+        integer :: i, n
+
+        call tokenize(string, set, first, last)
+
+        n = size(first)
+
+        allocate(character(maxval(last - first) + 1) :: tokens(n))
+
+        do i = 1, n
+            tokens(i) = string(first(i):last(i))
+        end do
+
+        if (present(separator)) then
+            allocate(character(1) :: separator(n - 1))
+
+            do i = 1, n - 1
+                separator(i) = string(last(i) + 1:last(i) + 1)
+            end do
+        end if
+    end subroutine tokenize_into_tokens_separator
 
 end module string_core_utils

src/dynamics/mpas/driver/dyn_mpas_procedures.F90

@@ -21,7 +21,9 @@ module dyn_mpas_procedures
     public :: almost_equal
     public :: clamp
     public :: index_unique
+    public :: split
     public :: stringify
+    public :: tokenize
 
     interface almost_divisible
         module procedure almost_divisible_real32
@@ -39,6 +41,11 @@ module dyn_mpas_procedures
         module procedure clamp_real32
         module procedure clamp_real64
     end interface clamp
+
+    interface tokenize
+        module procedure tokenize_into_first_last
+        module procedure tokenize_into_tokens_separator
+    end interface tokenize
 contains
     !> Test if `a` is divisible by `b`, where `a` and `b` are both reals.
     !> (KCW, 2024-05-25)
@@ -297,6 +304,47 @@ pure function index_unique(array)
         index_unique = pack([(i, i = 1, n)], mask_unique)
     end function index_unique
 
+    !> Parse a string into tokens, one at a time. Each character in `set` is a token delimiter.
+    !> If `back` is absent or is present with the value `.false.`, `pos` is assigned the position of the leftmost
+    !> token delimiter in `string` whose position is greater than `pos`, or if there is no such character, it
+    !> is assigned a value one greater than the length of `string`. This identifies a token with starting
+    !> position one greater than the value of `pos` on invocation, and ending position one less than the
+    !> value of `pos` on return.
+    !> If `back` is present with the value `.true.`, `pos` is assigned the position of the rightmost token delimiter
+    !> in `string` whose position is less than `pos`, or if there is no such character, it is assigned the value
+    !> zero. This identifies a token with ending position one less than the value of `pos` on invocation, and
+    !> starting position one greater than the value of `pos` on return.
+    !> This subroutine implements the `split` intrinsic procedure as defined in the Fortran 2023 language standard
+    !> (Section 16.9.196). We implement it ourselves because the compiler support may take years to become widespread.
+    !> (KCW, 2025-10-29)
+    pure subroutine split(string, set, pos, back)
+        character(*), intent(in) :: string, set
+        integer, intent(inout) :: pos
+        logical, optional, intent(in) :: back
+
+        integer :: offset
+
+        if (present(back)) then
+            if (back) then
+                offset = clamp(pos, 1, len(string) + 1)
+                pos = scan(string(1:offset - 1), set, back=.true.)
+
+                return
+            end if
+        end if
+
+        offset = clamp(pos, 0, len(string))
+        pos = scan(string(offset + 1:), set)
+
+        if (pos == 0) then
+            pos = len(string) + 1
+
+            return
+        end if
+
+        pos = offset + pos
+    end subroutine split
+
     !> Convert one or more values of any intrinsic data types to a character string for pretty printing.
     !> If `value` contains more than one element, the elements will be stringified, delimited by `separator`, then concatenated.
     !> If `value` contains exactly one element, the element will be stringified without using `separator`.
@@ -407,4 +455,73 @@ pure function stringify(value, separator)
 
         stringify = trim(buffer)
     end function stringify
+
+    !> Parse a string into tokens. Each character in `set` is a token delimiter.
+    !> `first` is allocated with the lower bound equal to one and the upper bound equal to the number of tokens in `string`.
+    !> Each element is assigned, in array element order, the starting position of each token in `string`, in the order found.
+    !> `last` is allocated with the lower bound equal to one and the upper bound equal to the number of tokens in `string`.
+    !> Each element is assigned, in array element order, the ending position of each token in `string`, in the order found.
+    !> This subroutine implements the `tokenize` intrinsic procedure as defined in the Fortran 2023 language standard
+    !> (Section 16.9.210). We implement it ourselves because the compiler support may take years to become widespread.
+    !> (KCW, 2025-10-29)
+    pure subroutine tokenize_into_first_last(string, set, first, last)
+        character(*), intent(in) :: string, set
+        integer, allocatable, intent(out) :: first(:), last(:)
+
+        integer :: pos_start(len(string) + 1), pos_end(len(string) + 1)
+        integer :: l, n, pos
+
+        l = len(string)
+        n = 0
+        pos = 0
+
+        do while (pos < l + 1)
+            n = n + 1
+            pos_start(n) = pos + 1
+
+            call split(string, set, pos)
+
+            pos_end(n) = pos - 1
+        end do
+
+        allocate(first(n), last(n))
+
+        first(:) = pos_start(1:n)
+        last(:) = pos_end(1:n)
+    end subroutine tokenize_into_first_last
+
+    !> Parse a string into tokens. Each character in `set` is a token delimiter.
+    !> `tokens` is allocated with the lower bound equal to one and the upper bound equal to the number of tokens in `string`,
+    !> and with character length equal to the length of the longest token. It contains the tokens in `string`.
+    !> `separator` is allocated with the lower bound equal to one and the upper bound equal to one less than the number of
+    !> tokens in `string`, and with character length equal to one. It contains the token delimiters in `string`.
+    !> This subroutine implements the `tokenize` intrinsic procedure as defined in the Fortran 2023 language standard
+    !> (Section 16.9.210). We implement it ourselves because the compiler support may take years to become widespread.
+    !> (KCW, 2025-10-29)
+    pure subroutine tokenize_into_tokens_separator(string, set, tokens, separator)
+        character(*), intent(in) :: string, set
+        character(:), allocatable, intent(out) :: tokens(:)
+        character(:), allocatable, optional, intent(out) :: separator(:)
+
+        integer, allocatable :: first(:), last(:)
+        integer :: i, n
+
+        call tokenize(string, set, first, last)
+
+        n = size(first)
+
+        allocate(character(maxval(last - first) + 1) :: tokens(n))
+
+        do i = 1, n
+            tokens(i) = string(first(i):last(i))
+        end do
+
+        if (present(separator)) then
+            allocate(character(1) :: separator(n - 1))
+
+            do i = 1, n - 1
+                separator(i) = string(last(i) + 1:last(i) + 1)
+            end do
+        end if
+    end subroutine tokenize_into_tokens_separator
 end module dyn_mpas_procedures