Index out of bounds

[dbeurle]

I've come across a crash when using cpp-coveralls (latest version) to get the gcov output, filter it using lcov and upload the lcov output to coveralls with cpp-coveralls. The crash surfaces in parse_lcov_file_info of coverage.py, I've added some print statements for debugging:

def parse_lcov_file_info(args, filepath, line_iter, line_coverage_re, file_end_string):
    """ Parse the file content in lcov info file
     """
     coverage = []
     lines_covered = []
     for line in line_iter:
         if line != "end_of_record":
             line_coverage_match = line_coverage_re.match(line)
             if line_coverage_match:
                 line_no = line_coverage_match.group(1)
                 cov_count = int(line_coverage_match.group(2))
                 if args.max_cov_count:
                     if cov_count > args.max_cov_count:
                         cov_count = args.max_cov_count + 1
                 lines_covered.append((line_no, cov_count))
         else:
             break
 
     line_list = [line.rstrip('\n') for line in open(filepath, 'r')]
 
     num_code_lines = len(line_list)
 
     coverage = [None] * num_code_lines
 
     for line_covered in lines_covered:
         if int(line_covered[0]) - 1 < num_code_lines:
            coverage[int(line_covered[0]) - 1] = line_covered[1]
         else:
             
             print('an error occurred\n')
             print('num_code_lines', num_code_lines)
             print('line_covered', line_covered)
             print('line_list', line_list)
             print('coverage', coverage, '\n')
     
     return coverage

and for the else branch I get (for example)

an error occurred

num_code_lines 109
line_covered ('187', 0)
line_list ['', '#pragma once', '', '#include "exceptions.hpp"', '', '#include <type_traits>', '#include <cfenv>', '', '/// \\file runge_kutta_integration.hpp', '/// \\brief Second, fourth and fifth order explicit methods for time integration', '', 'namespace neon', '{', '/// Perform a second order Runge-Kutta step for the given type.', '/// The type \\p function must accept time and the value.', 'template <typename T, typename function>', 'T runge_kutta_second_order(double const t, double const dt, T const& y, function&& f)', '{', '    T const dy1 = dt * f(t, y);', '    T const dy2 = dt * f(t + dt, y + dy1);', '    return y + (dy1 + dy2) / 2.0;', '}', '', '/// Perform a fourth order Runge-Kutta step for the given type.', '/// The type \\p function must accept time and the value.', 'template <typename T, typename function>', 'T runge_kutta_fourth_order(double const t, double const dt, T const& y, function&& f)', '{', '    T const dy1 = dt * f(t, y);', '    T const dy2 = dt * f(t + dt / 2.0, y + dy1 / 2.0);', '    T const dy3 = dt * f(t + dt / 2.0, y + dy2 / 2.0);', '    T const dy4 = dt * f(t + dt, y + dy3);', '', '    return y + (dy1 + 2.0 * dy2 + 2.0 * dy3 + dy4) / 6.0;', '}', '', 'namespace detail', '{', '/// Abstraction to compute the residual or the absolution value depending on', '/// the expression type (scalar or vector).', 'template <typename left_expression, typename right_expression>', 'double residual(left_expression const& left, right_expression const& right)', '{', '    if constexpr (!std::is_floating_point<left_expression>::value', '                  && !std::is_floating_point<right_expression>::value)', '    {', '        return (left - right).norm();', '    }', '    else', '    {', '        return std::abs(left - right);', '    }', '}', '}', '', '/// Perform the Dorman-Prince 4th order embedded Runge-Kutta time discretisation', '/// using the specified error tolerance \\cite DormandPrince1980', '/// \\tparam Lambda that returns the right hand side of the ODE', '/// \\param f Function object', '/// \\param error_tolerance Error tolerance for adaptive method', 'template <typename T, typename function>', 'T runge_kutta_fourth_fifth_order(double t,', '                                 double dt,', '                                 T y,', '                                 function&& f,', '                                 double const error_tolerance = 1.0e-5)', '{', '    auto const end_time = t + dt;', '', '    std::feclearexcept(FE_ALL_EXCEPT);', '', '    // Perform substep iterations until convergence', '    while (t < end_time)', '    {', '        // Correct an overzealous time step from the previous iteration', '        dt = std::min(dt, end_time - t);', '', '        // clang-format off', '        T const k1 = dt * f(t, y);', '        T const k2 = dt * f(t + 1.0 / 5.0 * dt,  y +        1.0 / 5.0 * k1);', '        T const k3 = dt * f(t + 3.0 / 10.0 * dt, y +       3.0 / 40.0 * k1 +       9.0 / 40.0 * k2);', '        T const k4 = dt * f(t + 4.0 / 5.0 * dt,  y +      44.0 / 45.0 * k1 -      56.0 / 15.0 * k2 +       32.0 / 9.0 * k3);', '        T const k5 = dt * f(t + 8.0 / 9.0 * dt,  y + 19372.0 / 6561.0 * k1 - 25360.0 / 2187.0 * k2 + 64448.0 / 6561.0 * k3 - 212.0 / 729.0 * k4);', '        T const k6 = dt * f(t + dt,              y +  9017.0 / 3168.0 * k1 -     355.0 / 33.0 * k2 + 46732.0 / 5247.0 * k3 +  49.0 / 176.0 * k4 - 5103.0 / 18656.0 * k5);', '        T const k7 = dt * f(t + dt,              y +     35.0 / 384.0 * k1                         +   500.0 / 1113.0 * k3 + 125.0 / 192.0 * k4 -  2187.0 / 6784.0 * k5 + 11.0 / 84.0 * k6);', '', '        T const dy_trial =     35.0 / 384.0 * k1 +   500.0 / 1113.0 * k3 + 125.0 / 192.0 * k4 -    2187.0 / 6784.0 * k5 +    11.0 / 84.0 * k6;', '        T const dy_error = 5179.0 / 57600.0 * k1 + 7571.0 / 16695.0 * k3 + 393.0 / 640.0 * k4 - 92097.0 / 339200.0 * k5 + 187.0 / 2100.0 * k6 + 1.0 / 40.0 * k7;', '        // clang-format on', '', '        double const R = detail::residual(dy_error, dy_trial) / dt;', '', '        if (R <= error_tolerance)', '        {', '            y += dy_trial;', '            t += dt;', '        }', '        dt *= 0.84', '              * std::pow(error_tolerance / (R + 4.0 * std::numeric_limits<double>::epsilon()), 0.25);', '', '        if (std::fetestexcept(FE_INVALID))', '        {', '            std::feclearexcept(FE_ALL_EXCEPT);', '            throw computational_error("Floating point error in adaptive RK45 detected.\\n");', '        }', '    }', '    return y;', '}', '}']
coverage [None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, 8, None, 8, 8, 8, None, 0, None, 0, 0, 98172, None, 98176, 98172, 98176, 98176, 4, 98172, 0, None, 0, 0, 0, None, 4, None, 4, 4, 4, None, None, None, None, None, 8, None, 8, 8, 8, 8, None, 8, None, 0, None, 0, 0, 0, 0, None, 0, None, 4, None, 4, 4, 4, 4, None, 4, None, 0, None, 0, 0, 0, 0, None, 0, None, 4, None, 4, 4, 4, 4, None, 4, None, None, None, None, None, None, None, 44, None, None, None, None, 24, None, None] 

I'm not sure why this has found itself in this state, ideas? Should it be looking for the maximum in the first tuple rather than from the return statements in the file to use as allocation?

[chrisosaurus]

When you say it crashed before you added those lines, were you seeing any exception information? if so please attach it here or in a gist.

Are you able to provide us with a minimum example to recreate this error?

[dbeurle]

I can't recall getting any useful exception information. Let me get a MWE going and I'll post back.

[dbeurle]

The back trace from the script

Traceback (most recent call last):
  File ".local/bin/cpp-coveralls", line 11, in <module>
    load_entry_point('cpp-coveralls==0.4.0', 'console_scripts', 'cpp-coveralls')()
  File ".local/lib/python3.6/site-packages/cpp_coveralls/__init__.py", line 97, in run
    cov_report = coverage.collect(args)
  File ".local/lib/python3.6/site-packages/cpp_coveralls/coverage.py", line 405, in collect
    src_report['coverage'] = parse_lcov_file_info(args, abs_filepath, line_iter, line_coverage_re, "end_of_record")
  File ".local/lib/python3.6/site-packages/cpp_coveralls/coverage.py", line 312, in parse_lcov_file_info
    coverage[int(line_covered[0]) - 1] = line_covered[1]
IndexError: list assignment index out of range

Do you want me to upload the offending lcov file?

[chrisosaurus]

yes please, that would be super helpful.

[mrc0mmand]

Apologies for resurrecting such an old thread, but I just hit this issue on Arch Linux, and as I can provide the lcov file it might help a bit to find the root cause:

# pip3 show cpp-coveralls
Name: cpp-coveralls
Version: 0.4.2
Summary: Upload gcov to coveralls.io
Home-page: https://github.com/eddyxu/cpp-coveralls
Author: Lei Xu
Author-email: [email protected]
License: Apache License 2.0
Location: /usr/lib/python3.9/site-packages
Requires: requests, urllib3, future
Required-by: 

# coveralls --dryrun --lcov-file /var/tmp/systemd-test-TEST-01-BASIC/coverage-info 
Traceback (most recent call last):
  File "/usr/bin/coveralls", line 33, in <module>
    sys.exit(load_entry_point('cpp-coveralls==0.4.2', 'console_scripts', 'coveralls')())
  File "/usr/lib/python3.9/site-packages/cpp_coveralls/__init__.py", line 97, in run
    cov_report = coverage.collect(args)
  File "/usr/lib/python3.9/site-packages/cpp_coveralls/coverage.py", line 413, in collect
    src_report['coverage'] = parse_lcov_file_info(args, abs_filepath, line_iter, line_coverage_re, "end_of_record")
  File "/usr/lib/python3.9/site-packages/cpp_coveralls/coverage.py", line 320, in parse_lcov_file_info
    coverage[int(line_covered[0]) - 1] = line_covered[1]
IndexError: list assignment index out of range

The lcov file seems to be correct:

# lcov -l /var/tmp/systemd-test-TEST-01-BASIC/coverage-info
Reading tracefile /var/tmp/systemd-test-TEST-01-BASIC/coverage-info
                                               |Lines      |Functions|Branches  
Filename                                       |Rate    Num|Rate  Num|Rate   Num
================================================================================
/build/src/basic/MurmurHash2.c                 | 0.0%    24| 0.0%   1|    -    0
/build/src/basic/af-list.c                     |33.3%    18|20.0%   5|    -    0
/build/src/basic/alloc-util.c                  | 100%    41| 100%   4|    -    0
/build/src/basic/alloc-util.h                  |81.2%    16|33.3%   3|    -    0
/build/src/basic/architecture.c                |91.7%    12|66.7%   3|    -    0
/build/src/basic/arphrd-list.c                 | 0.0%     6| 0.0%   1|    -    0
/build/src/basic/async.c                       |51.1%    45|50.0%   4|    -    0
/build/src/basic/audit-util.c                  |57.5%    40|66.7%   3|    -    0
/build/src/basic/audit-util.h                  | 100%     2|    -   0|    -    0
<...big snip...>
/build/src/systemd/sd-journal.h                | 100%     1|    -   0|    -    0
/build/src/systemd/sd-netlink.h                | 100%     2|    -   0|    -    0
/build/src/systemd/sd-resolve.h                | 0.0%     2|    -   0|    -    0
/build/src/sysusers/sysusers.c                 |59.5%  1100|89.8%  49|    -    0
/build/src/sysv-generator/sysv-generator.c     |16.3%   491|41.2%  17|    -    0
/build/src/tmpfiles/offline-passwd.c           | 0.0%    83| 0.0%   6|    -    0
/build/src/tmpfiles/tmpfiles.c                 |41.9%  1826|63.5%  85|    -    0
/build/src/update-done/update-done.c           |77.3%    22| 100%   2|    -    0
/build/src/update-utmp/update-utmp.c           |76.5%   102| 100%   8|    -    0
/build/src/user-sessions/user-sessions.c       |88.2%    17| 100%   2|    -    0
/build/src/veritysetup/veritysetup-generator.c |20.9%   225|90.9%  11|    -    0
/systemd-meson-build/s...sic/af-from-name.gperf| 100%    28| 100%   3|    -    0
/systemd-meson-build/s...arphrd-from-name.gperf| 0.0%    19| 0.0%   3|    -    0
/systemd-meson-build/src/basic/arphrd-to-name.h| 0.0%    68| 0.0%   1|    -    0
/systemd-meson-build/s...ic/cap-from-name.gperf| 100%    28| 100%   3|    -    0
/systemd-meson-build/s.../errno-from-name.gperf|93.3%    30| 100%   3|    -    0
/systemd-meson-build/s..._fs/restrict-fs-skel.h| 3.7%    54|33.3%   3|    -    0
/systemd-meson-build/s...restrict-ifaces.skel.h|78.3%    60| 100%   3|    -    0
/systemd-meson-build/s...ind/socket-bind.skel.h|78.0%    59| 100%   3|    -    0
/systemd-meson-build/s...d-fragment-gperf.gperf| 100%    28| 100%   2|    -    0
/systemd-meson-build/s...d/audit_type-to-name.h| 0.0%   216| 0.0%   1|    -    0
/systemd-meson-build/s...otocol-from-name.gperf| 0.0%    27| 0.0%   3|    -    0
================================================================================
                                         Total:|28.0%  139k|39.4%  8k|    -    0

lcov file: https://mrc0mmand.fedorapeople.org/systemd-TEST-01.coverage-info

[mrc0mmand]

The culprit appear to be gperf files:

# coveralls --dryrun --verbose --no-gcov --lcov-file /var/tmp/systemd-test-TEST-01-BASIC/coverage-info 
encodings: ['utf-8', 'latin-1']
Exception in /build/src/journal/journald-gperf.gperf
line_covered: ('58', 0)
coverage list len: 53
accessed index: 57
[None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, 0, None, 0, None, None, None, None, None, None, None, None, None, None, None, 0, None, 0, None, 0, None, 0, None, 0, 0, None, None, None]

which is, well, expected I guess, since the file in question indeed has only 53 lines:

# wc -l /build/src/journal/journald-gperf.gperf
53 /build/src/journal/journald-gperf.gperf

However, after ignoring the exception, it looks like the same issue affects most (if not all) other gperf files: https://gist.github.com/mrc0mmand/894b92df8b65ad43170319c1850b85bf