Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] add a duration custom option type to support arbitrary duration units #21307

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/python/pants/bin/local_pants_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import logging
import sys
from dataclasses import dataclass
from datetime import timedelta
from typing import Any

from pants.base.exiter import PANTS_FAILED_EXIT_CODE, PANTS_SUCCEEDED_EXIT_CODE, ExitCode
Expand Down Expand Up @@ -53,7 +54,7 @@ class LocalPantsRunner:

options: Options
options_bootstrapper: OptionsBootstrapper
session_end_tasks_timeout: float
session_end_tasks_timeout: timedelta
build_config: BuildConfiguration
run_tracker: RunTracker
specs: Specs
Expand Down
4 changes: 2 additions & 2 deletions src/python/pants/engine/internals/native_engine.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from __future__ import annotations

from datetime import datetime
from datetime import datetime, timedelta
from io import RawIOBase
from typing import (
Any,
Expand Down Expand Up @@ -863,7 +863,7 @@ def session_record_test_observation(
) -> None: ...
def session_isolated_shallow_clone(session: PySession, build_id: str) -> PySession: ...
def session_wait_for_tail_tasks(
scheduler: PyScheduler, session: PySession, timeout: float
scheduler: PyScheduler, session: PySession, timeout: timedelta
) -> None: ...
def graph_len(scheduler: PyScheduler) -> int: ...
def graph_visualize(scheduler: PyScheduler, session: PySession, path: str) -> None: ...
Expand Down
3 changes: 2 additions & 1 deletion src/python/pants/engine/internals/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import os
import time
from dataclasses import dataclass
from datetime import timedelta
from pathlib import PurePath
from types import CoroutineType
from typing import Any, Callable, Dict, Iterable, NoReturn, Sequence, cast
Expand Down Expand Up @@ -661,7 +662,7 @@ def is_cancelled(self) -> bool:
def cancel(self) -> None:
self.py_session.cancel()

def wait_for_tail_tasks(self, timeout: float) -> None:
def wait_for_tail_tasks(self, timeout: timedelta) -> None:
native_engine.session_wait_for_tail_tasks(self.py_scheduler, self.py_session, timeout)


Expand Down
39 changes: 39 additions & 0 deletions src/python/pants/option/custom_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import os
import re
import shlex
from datetime import timedelta
from enum import Enum
from typing import Iterable, Pattern, Sequence

Expand Down Expand Up @@ -167,6 +168,44 @@ def convert_to_bytes(power_of_2) -> int:
raise invalid


_DURATION_STR_RE = re.compile(r"^\s*([0-9.]+)\s*(\w*)\s*$")


def duration(val: timedelta | str | int | float) -> timedelta:
if isinstance(val, timedelta):
return val

if isinstance(val, (float, int)):
if val <= 0:
raise ParseError("A duration must be positive.")
return timedelta(seconds=val)

# Is the value in the form of `NUMBER UNIT`?
m = _DURATION_STR_RE.fullmatch(val)
if not m:
raise ParseError(f"The duration value `{val}` must be in `NUMBER UNIT` format.")

magnitude = int(m.group(1)) if "." not in m.group(1) else float(m.group(1))

maybe_unit = m.group(2)
unit = maybe_unit if maybe_unit else "s"

if unit in ("s", "second", "seconds"):
key = "seconds"
elif unit in ("ms", "milli", "millis", "millisecond", "milliseconds"):
key = "milliseconds"
elif unit in ("us", "micro", "micros", "microsecond", "microseconds"):
key = "microseconds"
elif unit in ("m", "minute", "minutes"):
key = "minutes"
elif unit in ("h", "hour", "hours"):
key = "hours"
else:
raise ParseError(f"Did not recognize the duration unit `{unit}`.")

return timedelta(**{key: magnitude})


def _convert(val, acceptable_types):
"""Ensure that val is one of the acceptable types, converting it if needed.

Expand Down
30 changes: 30 additions & 0 deletions src/python/pants/option/custom_types_test.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from datetime import timedelta
from textwrap import dedent
from typing import Dict, List, Union

Expand All @@ -11,6 +12,7 @@
ListValueComponent,
UnsetBool,
_flatten_shlexed_list,
duration,
memory_size,
)
from pants.option.errors import ParseError
Expand Down Expand Up @@ -64,6 +66,34 @@ def test_flatten_shlexed_list() -> None:
]


@pytest.mark.parametrize(
"unit,key",
[
("s", "seconds"),
("second", "seconds"),
("seconds", "seconds"),
("ms", "milliseconds"),
("milli", "milliseconds"),
("millis", "milliseconds"),
("millisecond", "milliseconds"),
("milliseconds", "milliseconds"),
("us", "microseconds"),
("micro", "microseconds"),
("micros", "microseconds"),
("microsecond", "microseconds"),
("microseconds", "microseconds"),
("m", "minutes"),
("minute", "minutes"),
("minutes", "minutes"),
("h", "hours"),
("hour", "hours"),
("hours", "hours"),
],
)
def test_duration_accepts_units(unit, key) -> None:
assert duration(f"10{unit}") == timedelta(**{key: 10})


class TestCustomTypes:
@staticmethod
def assert_list_parsed(s: str, *, expected: ParsedList) -> None:
Expand Down
7 changes: 4 additions & 3 deletions src/python/pants/option/global_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import sys
import tempfile
from dataclasses import dataclass
from datetime import datetime
from datetime import datetime, timedelta
from enum import Enum
from pathlib import Path, PurePath
from typing import Any, Callable, Type, TypeVar, cast
Expand All @@ -35,6 +35,7 @@
BoolOption,
DictOption,
DirOption,
DurationOption,
EnumOption,
FloatOption,
IntOption,
Expand Down Expand Up @@ -1489,8 +1490,8 @@ class BootstrapOptions:
),
advanced=True,
)
session_end_tasks_timeout = FloatOption(
default=3.0,
session_end_tasks_timeout = DurationOption(
default=timedelta(seconds=3.0),
help=softwrap(
"""
The time in seconds to wait for still-running "session end" tasks to complete before finishing
Expand Down
13 changes: 13 additions & 0 deletions src/python/pants/option/option_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import inspect
from dataclasses import dataclass
from datetime import timedelta
from enum import Enum
from typing import Any, Callable, Generic, Iterator, TypeVar, Union, cast, overload

Expand Down Expand Up @@ -786,6 +787,18 @@ def _convert_(self, val: Any) -> dict[str, _ValueT]:
return cast("dict[str, _ValueT]", val)


# -----------------------------------------------------------------------------------------------
# Duration Concrete Option Classes
# -----------------------------------------------------------------------------------------------
_DurationDefault = TypeVar("_DurationDefault", timedelta, None)


class DurationOption(_OptionBase[timedelta, _DurationDefault]):
"""An option representing a time duration (e.g., 50 milliseconds or 30 microseconds)."""

option_type: Any = custom_types.duration


# -----------------------------------------------------------------------------------------------
# "Specialized" Concrete Option Classes
# -----------------------------------------------------------------------------------------------
Expand Down
13 changes: 12 additions & 1 deletion src/python/pants/option/option_types_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,29 @@
from __future__ import annotations

import unittest.mock
from datetime import timedelta
from enum import Enum
from types import SimpleNamespace
from typing import TYPE_CHECKING, Any

import pytest

from pants.option.custom_types import dir_option, file_option, memory_size, shell_str, target_option
from pants.option.custom_types import (
dir_option,
duration,
file_option,
memory_size,
shell_str,
target_option,
)
from pants.option.option_types import (
ArgsListOption,
BoolListOption,
BoolOption,
DictOption,
DirListOption,
DirOption,
DurationOption,
EnumListOption,
EnumOption,
FileListOption,
Expand Down Expand Up @@ -68,6 +77,7 @@ def opt_info(*names, **options):
(FileOption, "a str", ".", dict(type=file_option)),
(ShellStrOption, "a str", "", dict(type=shell_str)),
(MemorySizeOption, 20, 22, dict(type=memory_size)),
(DurationOption, "10s", timedelta(microseconds=10), dict(type=duration)),
# List options
(StrListOption, ["a str"], ["str1", "str2"], dict(type=list, member_type=str)),
(IntListOption, [10], [1, 2], dict(type=list, member_type=int)),
Expand Down Expand Up @@ -314,6 +324,7 @@ class MySubsystem(MySubsystemBase):
FileOption,
ShellStrOption,
MemorySizeOption,
DurationOption,
StrListOption,
IntListOption,
FloatListOption,
Expand Down
3 changes: 3 additions & 0 deletions src/python/pants/option/options_fingerprinter.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import json
import os
from collections import defaultdict
from datetime import timedelta
from enum import Enum
from hashlib import sha1

Expand All @@ -21,6 +22,8 @@ def default(self, o):
if isinstance(o, dict):
# Sort by key to ensure that we don't invalidate if the insertion order changes.
return {k: self.default(v) for k, v in sorted(o.items())}
if isinstance(o, timedelta):
return str(o)
return super().default(o)


Expand Down
6 changes: 3 additions & 3 deletions src/rust/engine/src/externs/interface.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ use pyo3::prelude::{
pyclass, pyfunction, pymethods, pymodule, wrap_pyfunction, PyModule, PyObject,
PyResult as PyO3Result, Python, ToPyObject,
};
use pyo3::types::{PyBytes, PyDict, PyList, PyTuple, PyType};
use pyo3::types::{PyBytes, PyDelta, PyDict, PyList, PyTuple, PyType};
use pyo3::{create_exception, IntoPy, PyAny, PyRef};
use regex::Regex;
use remote::remote_cache::RemoteCacheWarningsBehavior;
Expand Down Expand Up @@ -1354,10 +1354,10 @@ fn session_wait_for_tail_tasks(
py: Python,
py_scheduler: &PyScheduler,
py_session: &PySession,
timeout: f64,
timeout: &PyDelta,
) -> PyO3Result<()> {
let core = &py_scheduler.0.core;
let timeout = Duration::from_secs_f64(timeout);
let timeout = timeout.extract()?;
core.executor.enter(|| {
py.allow_threads(|| {
core.executor
Expand Down
Loading