Revert "more direct set up data model (#1193)"

This reverts commit 8855842.

.flake8

@@ -1,3 +1,3 @@
 [flake8]
 max-line-length=160
-extend-ignore = E203
+extend-ignore = E203

.pre-commit-config.yaml

@@ -8,8 +8,3 @@ repos:
     hooks:
       - id: isort
         name: isort (python)
-  - repo: https://github.com/pycqa/flake8
-    rev: "7.1.0"
-    hooks:
-      - id: flake8
-        args: [--config=.flake8]

CHANGELOG.md

@@ -1,8 +1,3 @@
-# 3.0.0
-- Development in python packaging made supporting the previous namespace approach for
-  variable plugins untenable. Since we had to redo the way we defined the data model, 
-  we took the opportunity to explicity instantiate variable objects. 
-
 # 2.0.6
 - fixed bug that was preventing learning of index predicates in Dedupe mode
 

benchmarks/benchmarks/canonical.py

@@ -32,19 +32,17 @@ def make_report(self, clustering):
         return make_report(self.data, clustering)
 
     def run(self, use_settings=False):
-        deduper: dedupe.StaticDedupe | dedupe.Dedupe
-
         if use_settings and os.path.exists(self.settings_file):
             with open(self.settings_file, "rb") as f:
                 deduper = dedupe.StaticDedupe(f)
 
         else:
             variables = [
-                dedupe.variables.String("name"),
-                dedupe.variables.Exact("name"),
-                dedupe.variables.String("address"),
-                dedupe.variables.ShortString("cuisine", has_missing=True),
-                dedupe.variables.ShortString("city"),
+                {"field": "name", "type": "String"},
+                {"field": "name", "type": "Exact"},
+                {"field": "address", "type": "String"},
+                {"field": "cuisine", "type": "ShortString", "has missing": True},
+                {"field": "city", "type": "ShortString"},
             ]
 
             deduper = dedupe.Dedupe(variables, num_cores=5)

benchmarks/benchmarks/canonical_gazetteer.py

@@ -25,17 +25,16 @@ def make_report(self, clustering):
 
     def run(self, kwargs, use_settings=False):
         data_1, data_2 = self.data
-        gazetteer: dedupe.StaticGazetteer | dedupe.Gazetteer
 
         if use_settings and os.path.exists(self.settings_file):
             with open(self.settings_file, "rb") as f:
                 gazetteer = dedupe.StaticGazetteer(f)
         else:
             variables = [
-                dedupe.variables.String("name"),
-                dedupe.variables.String("address"),
-                dedupe.variables.String("cuisine"),
-                dedupe.variables.String("city"),
+                {"field": "name", "type": "String"},
+                {"field": "address", "type": "String"},
+                {"field": "cuisine", "type": "String"},
+                {"field": "city", "type": "String"},
             ]
 
             gazetteer = dedupe.Gazetteer(variables)

benchmarks/benchmarks/canonical_matching.py

@@ -42,17 +42,16 @@ def setup(self, kwargs):
 
     def run(self, kwargs, use_settings=False):
         data_1, data_2 = self.data
-        deduper: dedupe.StaticRecordLink | dedupe.RecordLink
 
         if use_settings and os.path.exists(self.settings_file):
             with open(self.settings_file, "rb") as f:
                 deduper = dedupe.StaticRecordLink(f)
         else:
             variables = [
-                dedupe.variables.String("name"),
-                dedupe.variables.String("address"),
-                dedupe.variables.String("cuisine"),
-                dedupe.variables.String("city"),
+                {"field": "name", "type": "String"},
+                {"field": "address", "type": "String"},
+                {"field": "cuisine", "type": "String"},
+                {"field": "city", "type": "String"},
             ]
             deduper = dedupe.RecordLink(variables)
             deduper.prepare_training(

benchmarks/benchmarks/common.py

@@ -54,9 +54,9 @@ def get_true_dupes(data: dict) -> set:
         sorted(data.items(), key=lambda x: x[1]["unique_id"]),
         key=lambda x: x[1]["unique_id"],
     ):
-        pair_l = list(pair)
-        if len(pair_l) == 2:
-            a, b = pair_l
+        pair = list(pair)
+        if len(pair) == 2:
+            a, b = pair
             duplicates.add(frozenset((a[0], b[0])))
     return duplicates
 

dedupe/__init__.py

@@ -13,18 +13,3 @@
     training_data_link,
 )
 from dedupe.serializer import read_training, write_training  # noqa: F401
-
-__all__ = [
-    "Dedupe",
-    "Gazetteer",
-    "RecordLink",
-    "StaticDedupe",
-    "StaticGazetteer",
-    "StaticRecordLink",
-    "canonicalize",
-    "console_label",
-    "training_data_dedupe",
-    "training_data_link",
-    "read_training",
-    "write_training",
-]

dedupe/_typing.py

@@ -4,18 +4,17 @@
     TYPE_CHECKING,
     Any,
     Callable,
+    Collection,
     Dict,
     FrozenSet,
     Iterable,
     Iterator,
     List,
     Mapping,
-    MutableSequence,
     Sequence,
     Tuple,
     Type,
     Union,
-    runtime_checkable,
 )
 
 import numpy
@@ -73,7 +72,6 @@
 LookupResults = Union[LookupResultsInt, LookupResultsStr]
 JoinConstraint = Literal["one-to-one", "many-to-one", "many-to-many"]
 Comparator = Callable[[Any, Any], Union[Union[int, float], Sequence[Union[int, float]]]]
-CustomComparator = Callable[[Any, Any], Union[int, float]]
 Scores = Union[numpy.memmap, numpy.ndarray]
 Labels = List[Literal[0, 1]]
 LabelsLike = Iterable[Literal[0, 1]]
@@ -83,10 +81,28 @@
 ComparisonCover = Union[ComparisonCoverInt, ComparisonCoverStr]
 PredicateFunction = Callable[[Any], FrozenSet[str]]
 
+VariableDefinition = TypedDict(
+    "VariableDefinition",
+    {
+        "type": str,
+        "field": str,
+        "variable name": str,
+        "corpus": Iterable[Union[str, Collection[str]]],
+        "comparator": Callable[
+            [Any, Any], Union[int, float]
+        ],  # a custom comparator can only return a single float or int, not a sequence of numbers
+        "categories": List[str],
+        "interaction variables": List[str],
+        "has missing": bool,
+        "name": str,
+    },
+    total=False,
+)
+
 
 class TrainingData(TypedDict):
-    match: MutableSequence[RecordDictPair]
-    distinct: MutableSequence[RecordDictPair]
+    match: List[RecordDictPair]
+    distinct: List[RecordDictPair]
 
 
 # Takes pairs of records and generates a (n_samples X n_features) array
@@ -111,24 +127,6 @@ def close(self) -> None: ...
     def join(self) -> None: ...
 
 
-class Variable(Protocol):
-    name: str
-    predicates: List["Predicate"]
-    has_missing: bool
-
-    def __len__(self) -> int: ...
-
-
-@runtime_checkable
-class FieldVariable(Variable, Protocol):
-    field: str
-    comparator: Comparator
-
-
-class InteractionVariable(Variable, Protocol):
-    interaction_fields: List[str]
-
-
 MapLike = Callable[[Callable[[Any], Any], Iterable], Iterable]
 
 PathLike = Union[str, os.PathLike]

dedupe/api.py

@@ -14,7 +14,7 @@
 import sqlite3
 import tempfile
 import warnings
-from typing import TYPE_CHECKING, Literal, cast, overload
+from typing import TYPE_CHECKING, cast, overload
 
 import numpy
 import sklearn.linear_model
@@ -27,6 +27,7 @@
 import dedupe.labeler as labeler
 import dedupe.predicates
 import dedupe.serializer as serializer
+from dedupe._typing import Literal
 
 if TYPE_CHECKING:
     from typing import (
@@ -69,7 +70,7 @@
         Scores,
         TrainingData,
         TupleLinks,
-        Variable,
+        VariableDefinition,
     )
 
 logger = logging.getLogger(__name__)
@@ -1116,7 +1117,7 @@ class ActiveMatching(Matching):
 
     def __init__(
         self,
-        variable_definition: Collection[Variable],
+        variable_definition: Collection[VariableDefinition],
         num_cores: int | None = None,
         in_memory: bool = False,
         **kwargs,

dedupe/convenience.py

@@ -7,14 +7,15 @@
 import random
 import sys
 import warnings
-from typing import Iterator, Literal, Tuple, overload
+from typing import Iterator, Tuple, overload
 
 import numpy
 
 import dedupe
 from dedupe._typing import (
     DataInt,
     DataStr,
+    Literal,
     RecordDict,
     RecordDictPair,
     RecordID,
@@ -134,7 +135,7 @@ def console_label(deduper: dedupe.api.ActiveMatching) -> None:  # pragma: no cov
 
     finished = False
     use_previous = False
-    fields = unique(var.field for var in deduper.data_model.field_variables)
+    fields = unique(var.field for var in deduper.data_model.primary_variables)
 
     buffer_len = 1  # Max number of previous operations
     unlabeled: list[RecordDictPair] = []

dedupe/core.py

@@ -22,7 +22,6 @@
         Generator,
         Iterable,
         Iterator,
-        Literal,
         Optional,
         Sequence,
         Type,
@@ -36,6 +35,7 @@
         ClosableJoinable,
         Data,
         FeaturizerFunction,
+        Literal,
         MapLike,
         RecordID,
         RecordIDDType,