diff --git a/fastcrawler/core/contracts.py b/fastcrawler/core/contracts.py new file mode 100644 index 0000000..5e48130 --- /dev/null +++ b/fastcrawler/core/contracts.py @@ -0,0 +1,24 @@ +from abc import ABC, abstractmethod + + +class ProcessItem(ABC): + + @property + def instances(self) -> list["ProcessItem"]: + if not hasattr(self, "_instances"): + self._instances = [self] + return self._instances + + def __rshift__(self, other: "ProcessItem") -> "ProcessItem": + """ + leveraged RSHIFT method for magic in flow >> + objA >> objB >> objC >> objD + """ + self.instances.append(other) + setattr(other, "instances", self.instances) + return other + + @abstractmethod + async def start(self): + ... + diff --git a/fastcrawler/core/spider.py b/fastcrawler/core/spider.py index 41b6254..dcc3ac7 100644 --- a/fastcrawler/core/spider.py +++ b/fastcrawler/core/spider.py @@ -13,8 +13,9 @@ from fastcrawler.parsers.schema import BaseModel from fastcrawler.utils.injection import _Depends +from .contracts import ProcessItem -class Spider: +class Spider(ProcessItem): """ Spider class to create the actual spider interface so that configuration of each spider can be given