Skip to content

Commit 820431a

Browse files
max-ostapenkomax-ostapenko
authored
Merge pull request #34 from HTTPArchive/development
Unique names and default sort for technologies and category
2 parents 08da3f5 + 9924880 commit 820431a

File tree

3 files changed

+126
-118
lines changed

3 files changed

+126
-118
lines changed

‎README.md

Lines changed: 66 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ Returns a JSON object with the following schema:
3939
"desktop": 11
4040
}
4141
},
42-
...
42+
...
4343
]
4444
```
4545

@@ -51,8 +51,8 @@ This endpoint can return a full list of categories names or a categories with al
5151

5252
The following parameters can be used to filter the data:
5353

54-
- `category` (`required`): A comma-separated string representing the category name(s).
55-
- `onlyname` (optional): A string 'true' or 'false'.
54+
- `category` (optional): A comma-separated string representing the category name(s).
55+
- `onlyname` (optional): No value required. If present, only the category names will be returned.
5656

5757
#### Response
5858

@@ -63,45 +63,46 @@ curl --request GET \
6363

6464
```json
6565
[
66-
{
67-
"description": "Solutions that redirect domains to a different location or page",
68-
"technologies": [
69-
"Arsys Domain Parking"
70-
],
71-
"origins": {
72-
"mobile": 14,
73-
"desktop": 8
74-
},
75-
"category": "Domain parking"
76-
},
77-
{
78-
"description": "Systems that automate building, testing, and deploying code",
79-
"technologies": [
80-
"Jenkins",
81-
"TeamCity"
82-
],
83-
"origins": {
84-
"mobile": 22,
85-
"desktop": 35
66+
{
67+
"description": "Systems that automate building, testing, and deploying code",
68+
"technologies": [
69+
"Jenkins",
70+
"TeamCity"
71+
],
72+
"origins": {
73+
"mobile": 22,
74+
"desktop": 35
75+
},
76+
"category": "CI"
8677
},
87-
"category": "CI"
88-
}
78+
{
79+
"description": "Solutions that redirect domains to a different location or page",
80+
"technologies": [
81+
"Cloudflare",
82+
"Arsys Domain Parking"
83+
],
84+
"origins": {
85+
"mobile": 14,
86+
"desktop": 8
87+
},
88+
"category": "Domain parking"
89+
}
8990
]
9091
```
9192

9293
```bash
9394
curl --request GET \
94-
--url 'https://{{HOST}}/v1/categories?onlyname=true'
95+
--url 'https://{{HOST}}/v1/categories?onlyname'
9596
```
9697

9798
```json
9899
[
99-
"Blogs",
100-
"LMS",
101-
"CI",
102-
"Cross border ecommerce",
103-
"Cart abandonment",
104-
"Domain parking",
100+
"A/B Testing",
101+
"Accessibility",
102+
"Accounting",
103+
"Advertising",
104+
"Affiliate programs",
105+
"Analytics",
105106
...
106107
]
107108

@@ -245,31 +246,50 @@ Returns a JSON object with the following schema:
245246

246247
The following parameters can be used to filter the data:
247248

248-
- `technology` (`required`): A comma-separated string representing the technology name(s).
249-
- `start` (optional): A string representing the start date in the format `YYYY-MM-DD`.
250-
- `end` (optional): A string representing the end date in the format `YYYY-MM-DD`.
251-
- `geo` (optional): A string representing the geographic location.
252-
- `rank` (optional): An string representing the rank.
249+
- `client` (optional): A string with the client: `mobile` or `desktop`.
250+
- `technology` (optional): A comma-separated string representing the technology name(s) or `ALL`.
253251
- `category` (optional): A comma-separated string representing the category name(s).
252+
- `onlyname` (optional): No value required. If present, only the technology names will be returned.
254253

255254
#### Response
256255

257256
```bash
258257
curl --request GET \
259-
--url 'https://{{HOST}}/v1/technologies?start=2022-02-01&end=2022-04-01&category=Live%20chat%2C%20blog&technology=Smartsupp&client=mobile'
258+
--url 'https://{{HOST}}/v1/technologies?category=Live%20chat%2C%20blog&technology=Smartsupp&client=mobile'
260259
```
261260

262261
Returns a JSON object with the following schema:
263262

264263
```json
265264
[
266-
{
267-
"client": "mobile",
268-
"similar_technologies": null,
269-
"description": "Smartsupp is a live chat tool that offers visitor recording feature.",
270-
"origins": 25649,
271-
"technology": "Smartsupp",
272-
"category": "Live chat"
273-
}
265+
{
266+
"client": "mobile",
267+
"similar_technologies": null,
268+
"description": "Smartsupp is a live chat tool that offers visitor recording feature.",
269+
"origins": 25649,
270+
"technology": "Smartsupp",
271+
"category": "Live chat"
272+
}
274273
]
275274
```
275+
276+
```bash
277+
curl --request GET \
278+
--url 'https://{{HOST}}/v1/technologies?onlyname'
279+
```
280+
281+
Returns a JSON object with the following schema:
282+
283+
```json
284+
[
285+
"1C-Bitrix",
286+
"2B Advice",
287+
"33Across",
288+
"34SP.com",
289+
"4-Tell",
290+
"42stores",
291+
"51.LA",
292+
"5centsCDN",
293+
...
294+
}
295+
```

‎functions/categories/libs/queries.py

Lines changed: 20 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,34 @@
11
import os
22
from google.cloud import firestore
3-
from google.cloud.firestore_v1.base_query import FieldFilter
4-
from .result import Result
3+
from google.cloud.firestore_v1.base_query import FieldFilter, Or
4+
from .result import Result
55
from .utils import convert_to_array
66

7-
DB = firestore.Client(project=os.environ.get('PROJECT'), database=os.environ.get('DATABASE'))
8-
TABLE = 'categories'
7+
DB = firestore.Client(
8+
project=os.environ.get("PROJECT"), database=os.environ.get("DATABASE")
9+
)
910

1011
def list_data(params):
11-
ref = DB.collection(TABLE)
12+
ref = DB.collection("categories")
1213

13-
query = ref
14+
query = ref.order_by("category", direction=firestore.Query.ASCENDING)
1415

15-
data = []
16+
if "category" in params:
17+
category_array = convert_to_array(params["category"])
18+
filter_array = []
19+
for category in category_array:
20+
filter_array.append(FieldFilter("category", "==", category))
21+
query = query.where(filter=Or(filters=filter_array))
1622

17-
if 'onlyname' in params:
1823
documents = query.stream()
24+
data = []
1925

20-
for doc in documents:
21-
item = doc.to_dict()
22-
if 'category' in item:
23-
data.append(item['category'])
24-
25-
else:
26-
27-
if 'category' in params:
28-
category_array = convert_to_array(params['category'])
29-
30-
for category in category_array:
31-
results = query.where(filter=FieldFilter("category", "==", category)).stream()
32-
for doc in results:
33-
data.append(doc.to_dict())
26+
if "onlyname" in params:
27+
for doc in documents:
28+
data.append(doc.get("category"))
3429

3530
else:
36-
documents = query.stream()
37-
38-
for doc in documents:
39-
data.append(doc.to_dict())
31+
for doc in documents:
32+
data.append(doc.to_dict())
4033

41-
return Result(result=data)
34+
return Result(result=data)

‎functions/technologies/libs/queries.py

Lines changed: 40 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -3,53 +3,48 @@
33
from google.cloud import firestore
44
from google.cloud.firestore_v1.base_query import FieldFilter, Or
55

6-
from .result import Result
6+
from .result import Result
77
from .utils import convert_to_array
88
from .presenters import Presenters
99

10-
DB = firestore.Client(project=os.environ.get('PROJECT'), database=os.environ.get('DATABASE'))
10+
DB = firestore.Client(
11+
project=os.environ.get("PROJECT"), database=os.environ.get("DATABASE")
12+
)
1113

1214
def list_data(params):
13-
onlyname = False
14-
ref = DB.collection('technologies')
15-
16-
query = ref
17-
18-
if 'technology' in params:
19-
arfilters = []
20-
params_array = convert_to_array(params['technology'])
21-
for tech in params_array:
22-
arfilters.append(FieldFilter('technology', '==', tech))
23-
24-
or_filter = Or(filters=arfilters)
25-
26-
query = query.where(filter=or_filter)
27-
28-
if 'category' in params:
29-
params_array = convert_to_array(params['category'])
30-
query = query.where(filter=FieldFilter('category_obj', 'array_contains_any', params_array))
31-
32-
if 'client' in params:
33-
query = query.where(filter=FieldFilter('client', '==', params['client']))
34-
35-
if 'onlyname' in params:
36-
onlyname = True
37-
38-
if 'sort' not in params:
39-
query = query.order_by('technology', direction=firestore.Query.ASCENDING)
40-
else:
41-
if params['sort'] == 'origins':
42-
query = query.order_by('origins', direction=firestore.Query.DESCENDING)
43-
44-
45-
documents = query.stream()
46-
47-
data = []
48-
for doc in documents:
49-
item = doc.to_dict()
50-
if onlyname:
51-
data.append(item['technology'])
52-
else:
53-
data.append(Presenters.technology(doc.to_dict()))
54-
55-
return Result(result=data)
15+
ref = DB.collection("technologies")
16+
17+
query = ref.order_by("technology", direction=firestore.Query.ASCENDING)
18+
19+
if "technology" in params:
20+
arfilters = []
21+
params_array = convert_to_array(params["technology"])
22+
for tech in params_array:
23+
arfilters.append(FieldFilter("technology", "==", tech))
24+
query = query.where(filter=Or(filters=arfilters))
25+
26+
if "category" in params:
27+
params_array = convert_to_array(params["category"])
28+
query = query.where(
29+
filter=FieldFilter("category_obj", "array_contains_any", params_array)
30+
)
31+
32+
if "client" in params:
33+
query = query.where(filter=FieldFilter("client", "==", params["client"]))
34+
35+
documents = query.stream()
36+
data = []
37+
38+
if "onlyname" in params and "client" not in params:
39+
appended_tech = set()
40+
for doc in documents:
41+
tech = doc.get("technology")
42+
if tech not in appended_tech:
43+
appended_tech.add(tech)
44+
data.append(tech)
45+
46+
else:
47+
for doc in documents:
48+
data.append(Presenters.technology(doc.to_dict()))
49+
50+
return Result(result=data)

0 commit comments

Comments
 (0)