diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 10ae7d77..32020576 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/psf/black - rev: 23.12.0 + rev: 24.4.2 hooks: - id: black - repo: https://github.com/pycqa/isort diff --git a/coverage.xml b/coverage.xml new file mode 100644 index 00000000..ec156e1f --- /dev/null +++ b/coverage.xml @@ -0,0 +1,2441 @@ + + + + + + /Users/fgregg/Developer/dedupe/dedupe + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dedupe/_typing.py b/dedupe/_typing.py index 4cc616f3..1d90e59b 100644 --- a/dedupe/_typing.py +++ b/dedupe/_typing.py @@ -114,21 +114,17 @@ class TrainingData(TypedDict): class Classifier(Protocol): """Takes an array of pairwise distances and computes the likelihood they are a pair.""" - def fit(self, X: numpy.typing.NDArray[numpy.float_], y: LabelsLike) -> None: - ... + def fit(self, X: numpy.typing.NDArray[numpy.float_], y: LabelsLike) -> None: ... def predict_proba( self, X: numpy.typing.NDArray[numpy.float_] - ) -> numpy.typing.NDArray[numpy.float_]: - ... + ) -> numpy.typing.NDArray[numpy.float_]: ... class ClosableJoinable(Protocol): - def close(self) -> None: - ... + def close(self) -> None: ... - def join(self) -> None: - ... + def join(self) -> None: ... MapLike = Callable[[Callable[[Any], Any], Iterable], Iterable] diff --git a/dedupe/api.py b/dedupe/api.py index 776cad01..1e2e1f43 100644 --- a/dedupe/api.py +++ b/dedupe/api.py @@ -206,13 +206,15 @@ def partition(self, data, threshold=0.5): # pragma: no cover @overload @staticmethod - def _add_singletons(all_ids: Iterable[int], clusters: ClustersInt) -> ClustersInt: - ... + def _add_singletons( + all_ids: Iterable[int], clusters: ClustersInt + ) -> ClustersInt: ... @overload @staticmethod - def _add_singletons(all_ids: Iterable[str], clusters: ClustersStr) -> ClustersStr: - ... + def _add_singletons( + all_ids: Iterable[str], clusters: ClustersStr + ) -> ClustersStr: ... @staticmethod def _add_singletons(all_ids, clusters): @@ -694,12 +696,10 @@ def __del__(self) -> None: self._close() @overload - def index(self, data: DataInt) -> None: - ... + def index(self, data: DataInt) -> None: ... @overload - def index(self, data: DataStr) -> None: - ... + def index(self, data: DataStr) -> None: ... def index(self, data): # pragma: no cover """ @@ -786,12 +786,10 @@ def unindex(self, data): # pragma: no cover del self.indexed_data[k] @overload - def blocks(self, data: DataInt) -> BlocksInt: - ... + def blocks(self, data: DataInt) -> BlocksInt: ... @overload - def blocks(self, data: DataStr) -> BlocksStr: - ... + def blocks(self, data: DataStr) -> BlocksStr: ... def blocks(self, data): """ @@ -1009,14 +1007,12 @@ def search( @overload def _format_search_results( self, search_d: DataInt, results: ArrayLinks - ) -> LookupResultsInt: - ... + ) -> LookupResultsInt: ... @overload def _format_search_results( self, search_d: DataStr, results: ArrayLinks - ) -> LookupResultsStr: - ... + ) -> LookupResultsStr: ... def _format_search_results(self, search_d, results): seen: set[RecordID] = set() diff --git a/dedupe/core.py b/dedupe/core.py index a73c7401..a9ffd55b 100644 --- a/dedupe/core.py +++ b/dedupe/core.py @@ -307,13 +307,11 @@ def Enumerator(start: int = 0) -> collections.defaultdict[Any, int]: @overload -def sniff_id_type(ids: Sequence[tuple[int, int]]) -> Type[int]: - ... +def sniff_id_type(ids: Sequence[tuple[int, int]]) -> Type[int]: ... @overload -def sniff_id_type(ids: Sequence[tuple[str, str]]) -> tuple[Type[str], Literal[256]]: - ... +def sniff_id_type(ids: Sequence[tuple[str, str]]) -> tuple[Type[str], Literal[256]]: ... def sniff_id_type(ids: Sequence[tuple[RecordID, RecordID]]) -> RecordIDDType: diff --git a/dedupe/labeler.py b/dedupe/labeler.py index 52e87cd7..44dbbf67 100644 --- a/dedupe/labeler.py +++ b/dedupe/labeler.py @@ -258,12 +258,10 @@ def _index_predicates(self, candidates: TrainingExamples) -> None: pred.freeze(records) @overload - def _sample(self, data: DataInt, sample_size: int) -> TrainingExamples: - ... + def _sample(self, data: DataInt, sample_size: int) -> TrainingExamples: ... @overload - def _sample(self, data: DataStr, sample_size: int) -> TrainingExamples: - ... + def _sample(self, data: DataStr, sample_size: int) -> TrainingExamples: ... def _sample(self, data, sample_size): sample_indices = self._sample_indices( @@ -323,14 +321,12 @@ def _index_predicates(self, candidates: TrainingExamples) -> None: @overload def _sample( self, data_1: DataInt, data_2: DataInt, sample_size: int - ) -> TrainingExamples: - ... + ) -> TrainingExamples: ... @overload def _sample( self, data_1: DataStr, data_2: DataStr, sample_size: int - ) -> TrainingExamples: - ... + ) -> TrainingExamples: ... def _sample(self, data_1, data_2, sample_size): sample_indices = self._sample_indices(sample_size, len(data_1) * len(data_2)) diff --git a/dedupe/predicates.py b/dedupe/predicates.py index 571992b2..2d180ee8 100644 --- a/dedupe/predicates.py +++ b/dedupe/predicates.py @@ -151,19 +151,16 @@ def __setstate__(self, d: Mapping[str, Any]) -> None: self.index = None @abc.abstractmethod - def reset(self) -> None: - ... + def reset(self) -> None: ... @abc.abstractmethod - def initIndex(self) -> Index: - ... + def initIndex(self) -> Index: ... def bust_cache(self) -> None: self._cache = {} @abc.abstractmethod - def preprocess(self, doc: Any) -> Any: - ... + def preprocess(self, doc: Any) -> Any: ... class CanopyPredicate(IndexPredicate): diff --git a/dedupe/training.py b/dedupe/training.py index 56a07814..eb6fe0a0 100644 --- a/dedupe/training.py +++ b/dedupe/training.py @@ -201,15 +201,13 @@ def __init__( @staticmethod def coveredPairs( blocker: blocking.Fingerprinter, records: DataInt - ) -> ComparisonCoverInt: - ... + ) -> ComparisonCoverInt: ... @overload @staticmethod def coveredPairs( blocker: blocking.Fingerprinter, records: DataStr - ) -> ComparisonCoverStr: - ... + ) -> ComparisonCoverStr: ... @staticmethod def coveredPairs(blocker: blocking.Fingerprinter, records): @@ -251,8 +249,7 @@ def __init__( sampled_records_1: DataInt, sampled_records_2: DataInt, data_2: DataInt, - ): - ... + ): ... @overload def __init__( @@ -261,8 +258,7 @@ def __init__( sampled_records_1: DataStr, sampled_records_2: DataStr, data_2: DataStr, - ): - ... + ): ... def __init__( self, @@ -281,14 +277,12 @@ def __init__( @overload def coveredPairs( self, blocker: blocking.Fingerprinter, records_1: DataInt, records_2: DataInt - ) -> ComparisonCoverInt: - ... + ) -> ComparisonCoverInt: ... @overload def coveredPairs( self, blocker: blocking.Fingerprinter, records_1: DataStr, records_2: DataStr - ) -> ComparisonCoverStr: - ... + ) -> ComparisonCoverStr: ... def coveredPairs(self, blocker, records_1, records_2): cover: dict[Predicate, dict[str, tuple[set[RecordID], set[RecordID]]]] = {} diff --git a/pyproject.toml b/pyproject.toml index 508cac74..326f7099 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "scikit-learn", "affinegap>=1.3", "categorical-distance>=1.9", - "numpy>=1.20", + "numpy>=1.20,<2.0", "doublemetaphone", "highered>=0.2.0", "simplecosine>=1.2",