Skip to content

Commit

Permalink
Feat: support to generate pure ASN data
Browse files Browse the repository at this point in the history
when using maxmindGeoLite2ASNCSV as input format
  • Loading branch information
Loyalsoldier committed Jan 31, 2025
1 parent ebc3b25 commit d9ccff7
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 19 deletions.
52 changes: 50 additions & 2 deletions configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -328,10 +328,10 @@

- **type**:(必须)输入格式的名称
- **action**:(必须)操作类型,值为 `add`(添加 IP 地址)或 `remove`(移除 IP 地址)
- **args**:(必须
- **args**:(可选
- **ipv4**:(可选)MaxMind GeoLite2 ASN IPv4 文件路径(`GeoLite2-ASN-Blocks-IPv4.csv`),可为本地文件路径或远程 `http``https` 文件 URL。
- **ipv6**:(可选)MaxMind GeoLite2 ASN IPv6 文件路径(`GeoLite2-ASN-Blocks-IPv6.csv`),可为本地文件路径或远程 `http``https` 文件 URL。
- **wantedList**:(必须,对象,键为字符串类别名,值为 ASN 字符串数组)指定类别名及其包含的 ASN。
- **wantedList**:(可选,数组或对象;当为数组时,值为 ASN 字符串;当为对象时,键为类别名,值为 ASN 字符串数组)指定 ASN 或类别名及其包含的 ASN。若未指定,则默认选择所有 ASN。
- **onlyIPType**:(可选)只处理的 IP 地址类型,值为 `ipv4``ipv6`

```jsonc
Expand Down Expand Up @@ -366,6 +366,54 @@
}
```

```jsonc
// 由于未指定 `wantedList`,自动将所有 ASN 添加为类别,类别名格式为 AS + ASN 字符串,如 `AS123`、`AS12345`
{
"type": "maxmindGeoLite2ASNCSV",
"action": "add" // 添加 IP 地址
}
```

```jsonc
// 由于未指定 `wantedList`,自动将所有 ASN 添加为类别,类别名格式为 AS + ASN 字符串,如 `AS123`、`AS12345`
{
"type": "maxmindGeoLite2ASNCSV",
"action": "add", // 添加 IP 地址
"args": {
"onlyIPType": "ipv4" // 只添加各自的 IPv4 地址
}
}
```

```jsonc
// 由于未指定 `wantedList`,自动移除所有匹配的 ASN 类别,匹配的类别名格式为 AS + ASN 字符串,如 `AS123`、`AS12345`
{
"type": "maxmindGeoLite2ASNCSV",
"action": "remove" // 移除 IP 地址
}
```

```jsonc
{
"type": "maxmindGeoLite2ASNCSV",
"action": "add", // 添加 IP 地址
"args": {
"wantedList": ["AS123", "AS4567"] // 向名为 AS123 和 AS4567 的类别中分别添加各自的 IPv4 和 IPv6 地址
}
}
```

```jsonc
{
"type": "maxmindGeoLite2ASNCSV",
"action": "remove", // 移除 IP 地址
"args": {
"wantedList": ["AS123", "AS4567"], // 从名为 AS123 和 AS4567 的类别中分别移除各自的 IPv6 地址
"onlyIPType": "ipv6"
}
}
```

### **maxmindGeoLite2CountryCSV**

- **type**:(必须)输入格式的名称
Expand Down
28 changes: 28 additions & 0 deletions lib/common.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package lib

import (
"encoding/json"
"fmt"
"io"
"net/http"
Expand Down Expand Up @@ -32,3 +33,30 @@ func GetRemoteURLReader(url string) (io.ReadCloser, error) {

return resp.Body, nil
}

type WantedListExtended struct {
TypeSlice []string
TypeMap map[string][]string
}

func (w *WantedListExtended) UnmarshalJSON(data []byte) error {
if len(data) == 0 {
return nil
}

slice := make([]string, 0)
mapMap := make(map[string][]string, 0)

err := json.Unmarshal(data, &slice)
if err != nil {
err2 := json.Unmarshal(data, &mapMap)
if err2 != nil {
return err2
}
}

w.TypeSlice = slice
w.TypeMap = mapMap

return nil
}
60 changes: 43 additions & 17 deletions plugin/maxmind/maxmind_asn_csv_in.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,10 @@ func init() {

func newGeoLite2ASNCSV(action lib.Action, data json.RawMessage) (lib.InputConverter, error) {
var tmp struct {
IPv4File string `json:"ipv4"`
IPv6File string `json:"ipv6"`
Want map[string][]string `json:"wantedList"`
OnlyIPType lib.IPType `json:"onlyIPType"`
IPv4File string `json:"ipv4"`
IPv6File string `json:"ipv6"`
Want lib.WantedListExtended `json:"wantedList"`
OnlyIPType lib.IPType `json:"onlyIPType"`
}

if len(data) > 0 {
Expand All @@ -53,8 +53,9 @@ func newGeoLite2ASNCSV(action lib.Action, data json.RawMessage) (lib.InputConver
}

// Filter want list
wantList := make(map[string][]string) // map[asn][]listname
for list, asnList := range tmp.Want {
wantList := make(map[string][]string) // map[asn][]listname or map[asn][]asn

for list, asnList := range tmp.Want.TypeMap {
list = strings.ToUpper(strings.TrimSpace(list))
if list == "" {
continue
Expand All @@ -75,8 +76,13 @@ func newGeoLite2ASNCSV(action lib.Action, data json.RawMessage) (lib.InputConver
}
}

if len(wantList) == 0 {
return nil, fmt.Errorf("❌ [type %s | action %s] wantedList must be specified in config", TypeASNCSV, action)
for _, asn := range tmp.Want.TypeSlice {
asn = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(asn)), "as")
if asn == "" {
continue
}

wantList[asn] = []string{"AS" + asn}
}

return &GeoLite2ASNCSV{
Expand Down Expand Up @@ -192,16 +198,36 @@ func (g *GeoLite2ASNCSV) process(file string, entries map[string]*lib.Entry) err
return fmt.Errorf("❌ [type %s | action %s] invalid record: %v", g.Type, g.Action, record)
}

if listArr, found := g.Want[strings.TrimSpace(record[1])]; found {
for _, listName := range listArr {
entry, got := entries[listName]
if !got {
entry = lib.NewEntry(listName)
}
if err := entry.AddPrefix(strings.TrimSpace(record[0])); err != nil {
return err
// Maxmind ASN CSV reference:
// network,autonomous_system_number,autonomous_system_organization
// 1.0.0.0/24,13335,CLOUDFLARENET
// 1.0.4.0/22,38803,"Gtelecom Pty Ltd"
// 1.0.16.0/24,2519,"ARTERIA Networks Corporation"

switch len(g.Want) {
case 0: // it means user wants all ASNs
asn := "AS" + strings.TrimSpace(record[1]) // default list name is in "AS12345" format
entry, got := entries[asn]
if !got {
entry = lib.NewEntry(asn)
}
if err := entry.AddPrefix(strings.TrimSpace(record[0])); err != nil {
return err
}
entries[asn] = entry

default: // it means user wants specific ASNs or customized lists with specific ASNs
if listArr, found := g.Want[strings.TrimSpace(record[1])]; found {
for _, listName := range listArr {
entry, got := entries[listName]
if !got {
entry = lib.NewEntry(listName)
}
if err := entry.AddPrefix(strings.TrimSpace(record[0])); err != nil {
return err
}
entries[listName] = entry
}
entries[listName] = entry
}
}
}
Expand Down

0 comments on commit d9ccff7

Please sign in to comment.