apacheGH-47633: [Dev][Integration] Write all files with --write_generated_json (apache#47634)

### Rationale for this change

The `--write_generated_json` option to `archery integration` was originally a JS-specific hack, but its usage was removed from the JS codebase in commit [c65bcda60b899649c3daf6f16ff5c4e12a41f06d](apache/arrow-js@c65bcda).

### What changes are included in this PR?

Rework this option to write all generated JSON integration files, not a subset of them.

Also fix the option's spelling to use hyphens not underscores.

### Are these changes tested?

Only manually.

### Are there any user-facing changes?

No.
* GitHub Issue: apache#47633

Authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Antoine Pitrou <antoine@python.org>

Author: pitrou

dev/archery/archery/cli.py

@@ -691,7 +691,7 @@ def _set_default(opt, default):
 @click.option('--target-implementations', default='',
               help=('Target implementations in this integration tests'),
               envvar="ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS")
-@click.option('--write_generated_json', default="",
+@click.option('--write-generated-json', default="",
               help='Generate test JSON to indicated path')
 @click.option('--run-ipc', is_flag=True, default=False,
               help='Run IPC integration tests')
@@ -714,7 +714,8 @@ def _set_default(opt, default):
 @click.option('-k', '--match',
               help=("Substring for test names to include in run, "
                     "e.g. -k primitive"))
-def integration(with_all=False, random_seed=12345, **args):
+def integration(with_all=False, random_seed=12345, write_generated_json="",
+                **args):
     """If you don't specify the "--target-implementations" option nor
     the "ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS" environment
     variable, test patterns are product of all specified
@@ -773,7 +774,8 @@ def integration(with_all=False, random_seed=12345, **args):
 
     """
 
-    from .integration.runner import write_js_test_json, run_all_tests
+    from .integration.datagen import get_generated_json_files
+    from .integration.runner import run_all_tests
     import numpy as np
 
     # FIXME(bkietz) Include help strings for individual testers.
@@ -782,8 +784,6 @@ def integration(with_all=False, random_seed=12345, **args):
     # Make runs involving data generation deterministic
     np.random.seed(random_seed)
 
-    gen_path = args['write_generated_json']
-
     implementations = ['cpp', 'dotnet', 'java', 'js', 'go', 'nanoarrow', 'rust']
     formats = ['ipc', 'flight', 'c_data']
 
@@ -799,11 +799,9 @@ def integration(with_all=False, random_seed=12345, **args):
         param = f'run_{fmt}'
         enabled_formats += args[param]
 
-    if gen_path:
-        # XXX See GH-37575: this option is only used by the JS test suite
-        # and might not be useful anymore.
-        os.makedirs(gen_path, exist_ok=True)
-        write_js_test_json(gen_path)
+    if write_generated_json:
+        os.makedirs(write_generated_json, exist_ok=True)
+        get_generated_json_files(tempdir=write_generated_json)
     else:
         if enabled_formats == 0:
             raise click.UsageError(

dev/archery/archery/integration/runner.py

@@ -750,81 +750,3 @@ def append_tester(implementation, tester):
         log(f"{fail_count} failures, {len(runner.skips)} skips")
         if fail_count > 0:
             sys.exit(1)
-
-
-def write_js_test_json(directory):
-    datagen.generate_primitive_case([], name='primitive_no_batches').write(
-        os.path.join(directory, 'primitive-no-batches.json')
-    )
-    datagen.generate_primitive_case([17, 20], name='primitive').write(
-        os.path.join(directory, 'primitive.json')
-    )
-    datagen.generate_primitive_case([0, 0, 0], name='primitive_zerolength').write(
-        os.path.join(directory, 'primitive-empty.json')
-    )
-    # datagen.generate_primitive_large_offsets_case([17, 20]).write(
-    #     os.path.join(directory, 'primitive-large-offsets.json')
-    # )
-    datagen.generate_null_case([10, 0]).write(
-        os.path.join(directory, 'null.json')
-    )
-    datagen.generate_null_trivial_case([0, 0]).write(
-        os.path.join(directory, 'null-trivial.json')
-    )
-    datagen.generate_decimal128_case().write(
-        os.path.join(directory, 'decimal128.json')
-    )
-    # datagen.generate_decimal256_case().write(
-    #     os.path.join(directory, 'decimal256.json')
-    # )
-    datagen.generate_datetime_case().write(
-        os.path.join(directory, 'datetime.json')
-    )
-    # datagen.generate_duration_case().write(
-    #     os.path.join(directory, 'duration.json')
-    # )
-    # datagen.generate_interval_case().write(
-    #     os.path.join(directory, 'interval.json')
-    # )
-    # datagen.generate_month_day_nano_interval_case().write(
-    #     os.path.join(directory, 'month_day_nano_interval.json')
-    # )
-    datagen.generate_map_case().write(
-        os.path.join(directory, 'map.json')
-    )
-    datagen.generate_non_canonical_map_case().write(
-        os.path.join(directory, 'non_canonical_map.json')
-    )
-    datagen.generate_nested_case().write(
-        os.path.join(directory, 'nested.json')
-    )
-    datagen.generate_recursive_nested_case().write(
-        os.path.join(directory, 'recursive-nested.json')
-    )
-    # datagen.generate_nested_large_offsets_case().write(
-    #     os.path.join(directory, 'nested-large-offsets.json')
-    # )
-    datagen.generate_unions_case().write(
-        os.path.join(directory, 'unions.json')
-    )
-    datagen.generate_custom_metadata_case().write(
-        os.path.join(directory, 'custom-metadata.json')
-    )
-    # datagen.generate_duplicate_fieldnames_case().write(
-    #     os.path.join(directory, 'duplicate-fieldnames.json')
-    # )
-    datagen.generate_dictionary_case().write(
-        os.path.join(directory, 'dictionary.json')
-    )
-    datagen.generate_dictionary_unsigned_case().write(
-        os.path.join(directory, 'dictionary-unsigned.json')
-    )
-    datagen.generate_nested_dictionary_case().write(
-        os.path.join(directory, 'dictionary-nested.json')
-    )
-    # datagen.generate_run_end_encoded_case().write(
-    #     os.path.join(directory, 'run_end_encoded.json')
-    # )
-    datagen.generate_extension_case().write(
-        os.path.join(directory, 'extension.json')
-    )