From 7ace2d59a8c55da66acd3d1a378aac43cc5c2e38 Mon Sep 17 00:00:00 2001 From: Loyalsoldier <10487845+Loyalsoldier@users.noreply.github.com> Date: Sun, 21 Nov 2021 15:56:50 +0800 Subject: Feat: support v2rayGeoIPDat as input format --- plugin/v2ray/dat.go | 227 ------------------------------------------------ plugin/v2ray/dat_in.go | 204 +++++++++++++++++++++++++++++++++++++++++++ plugin/v2ray/dat_out.go | 227 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 431 insertions(+), 227 deletions(-) delete mode 100644 plugin/v2ray/dat.go create mode 100644 plugin/v2ray/dat_in.go create mode 100644 plugin/v2ray/dat_out.go (limited to 'plugin') diff --git a/plugin/v2ray/dat.go b/plugin/v2ray/dat.go deleted file mode 100644 index 0c4e76d8..00000000 --- a/plugin/v2ray/dat.go +++ /dev/null @@ -1,227 +0,0 @@ -package v2ray - -import ( - "encoding/json" - "fmt" - "log" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/Loyalsoldier/geoip/lib" - "github.com/v2fly/v2ray-core/v4/app/router" - "github.com/v2fly/v2ray-core/v4/infra/conf/rule" - "google.golang.org/protobuf/proto" -) - -const ( - typeGeoIPdat = "v2rayGeoIPDat" - descGeoIPdat = "Convert data to V2Ray GeoIP dat format" -) - -var ( - defaultOutputName = "geoip.dat" - defaultOutputDir = filepath.Join("./", "output", "dat") -) - -func init() { - lib.RegisterOutputConfigCreator(typeGeoIPdat, func(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { - return newGeoIPDat(action, data) - }) - lib.RegisterOutputConverter(typeGeoIPdat, &geoIPDat{ - Description: descGeoIPdat, - }) -} - -func newGeoIPDat(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { - var tmp struct { - OutputName string `json:"outputName"` - OutputDir string `json:"outputDir"` - Want []string `json:"wantedList"` - OneFilePerList bool `json:"oneFilePerList"` - OnlyIPType lib.IPType `json:"onlyIPType"` - } - - if len(data) > 0 { - if err := json.Unmarshal(data, &tmp); err != nil { - return nil, err - } - } - - if tmp.OutputName == "" { - tmp.OutputName = defaultOutputName - } - - if tmp.OutputDir == "" { - tmp.OutputDir = defaultOutputDir - } - - return &geoIPDat{ - Type: typeGeoIPdat, - Action: action, - Description: descGeoIPdat, - OutputName: tmp.OutputName, - OutputDir: tmp.OutputDir, - Want: tmp.Want, - OneFilePerList: tmp.OneFilePerList, - OnlyIPType: tmp.OnlyIPType, - }, nil -} - -type geoIPDat struct { - Type string - Action lib.Action - Description string - OutputName string - OutputDir string - Want []string - OneFilePerList bool - OnlyIPType lib.IPType -} - -func (g *geoIPDat) GetType() string { - return g.Type -} - -func (g *geoIPDat) GetAction() lib.Action { - return g.Action -} - -func (g *geoIPDat) GetDescription() string { - return g.Description -} - -func (g *geoIPDat) Output(container lib.Container) error { - // Filter want list - wantList := make(map[string]bool) - for _, want := range g.Want { - if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { - wantList[want] = true - } - } - - geoIPList := new(router.GeoIPList) - geoIPList.Entry = make([]*router.GeoIP, 0, 300) - updated := false - switch len(wantList) { - case 0: - for entry := range container.Loop() { - geoIP, err := g.generateGeoIP(entry) - if err != nil { - return err - } - geoIPList.Entry = append(geoIPList.Entry, geoIP) - updated = true - - if g.OneFilePerList { - geoIPBytes, err := proto.Marshal(geoIPList) - if err != nil { - return err - } - filename := strings.ToLower(entry.GetName()) + ".dat" - if err := g.writeFile(filename, geoIPBytes); err != nil { - return err - } - geoIPList.Entry = nil - } - } - - default: - for name := range wantList { - entry, found := container.GetEntry(name) - if !found { - log.Printf("❌ entry %s not found", name) - continue - } - geoIP, err := g.generateGeoIP(entry) - if err != nil { - return err - } - geoIPList.Entry = append(geoIPList.Entry, geoIP) - updated = true - - if g.OneFilePerList { - geoIPBytes, err := proto.Marshal(geoIPList) - if err != nil { - return err - } - filename := strings.ToLower(entry.GetName()) + ".dat" - if err := g.writeFile(filename, geoIPBytes); err != nil { - return err - } - geoIPList.Entry = nil - } - } - } - - // Sort to make reproducible builds - g.sort(geoIPList) - - if !g.OneFilePerList && updated { - geoIPBytes, err := proto.Marshal(geoIPList) - if err != nil { - return err - } - if err := g.writeFile(g.OutputName, geoIPBytes); err != nil { - return err - } - } - - return nil -} - -func (g *geoIPDat) generateGeoIP(entry *lib.Entry) (*router.GeoIP, error) { - var entryCidr []string - var err error - switch g.OnlyIPType { - case lib.IPv4: - entryCidr, err = entry.MarshalText(lib.IgnoreIPv6) - case lib.IPv6: - entryCidr, err = entry.MarshalText(lib.IgnoreIPv4) - default: - entryCidr, err = entry.MarshalText() - } - if err != nil { - return nil, err - } - - v2rayCIDR := make([]*router.CIDR, 0, 1024) - for _, cidrStr := range entryCidr { - cidr, err := rule.ParseIP(cidrStr) - if err != nil { - return nil, err - } - v2rayCIDR = append(v2rayCIDR, cidr) - } - - if len(v2rayCIDR) > 0 { - return &router.GeoIP{ - CountryCode: entry.GetName(), - Cidr: v2rayCIDR, - }, nil - } - - return nil, fmt.Errorf("entry %s has no CIDR", entry.GetName()) -} - -// Sort by country code to make reproducible builds -func (g *geoIPDat) sort(list *router.GeoIPList) { - sort.SliceStable(list.Entry, func(i, j int) bool { - return list.Entry[i].CountryCode < list.Entry[j].CountryCode - }) -} - -func (g *geoIPDat) writeFile(filename string, geoIPBytes []byte) error { - if err := os.MkdirAll(g.OutputDir, 0755); err != nil { - return err - } - - if err := os.WriteFile(filepath.Join(g.OutputDir, filename), geoIPBytes, 0644); err != nil { - return err - } - - log.Printf("✅ [%s] %s --> %s", g.Type, filename, g.OutputDir) - - return nil -} diff --git a/plugin/v2ray/dat_in.go b/plugin/v2ray/dat_in.go new file mode 100644 index 00000000..a8e1974f --- /dev/null +++ b/plugin/v2ray/dat_in.go @@ -0,0 +1,204 @@ +package v2ray + +import ( + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "strings" + + "github.com/Loyalsoldier/geoip/lib" + "github.com/v2fly/v2ray-core/v4/app/router" + "google.golang.org/protobuf/proto" +) + +const ( + typeGeoIPdatIn = "v2rayGeoIPDat" + descGeoIPdatIn = "Convert V2Ray GeoIP dat to other formats" +) + +func init() { + lib.RegisterInputConfigCreator(typeGeoIPdatIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + return newGeoIPDatIn(action, data) + }) + lib.RegisterInputConverter(typeGeoIPdatIn, &geoIPDatIn{ + Description: descGeoIPdatIn, + }) +} + +func newGeoIPDatIn(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + var tmp struct { + URI string `json:"uri"` + Want []string `json:"wantedList"` + OnlyIPType lib.IPType `json:"onlyIPType"` + } + + if len(data) > 0 { + if err := json.Unmarshal(data, &tmp); err != nil { + return nil, err + } + } + + if tmp.URI == "" { + return nil, fmt.Errorf("[type %s | action %s] uri must be specified in config", typeGeoIPdatIn, action) + } + + return &geoIPDatIn{ + Type: typeGeoIPdatIn, + Action: action, + Description: descGeoIPdatIn, + URI: tmp.URI, + Want: tmp.Want, + OnlyIPType: tmp.OnlyIPType, + }, nil +} + +type geoIPDatIn struct { + Type string + Action lib.Action + Description string + URI string + Want []string + OnlyIPType lib.IPType +} + +func (g *geoIPDatIn) GetType() string { + return g.Type +} + +func (g *geoIPDatIn) GetAction() lib.Action { + return g.Action +} + +func (g *geoIPDatIn) GetDescription() string { + return g.Description +} + +func (g *geoIPDatIn) Input(container lib.Container) (lib.Container, error) { + entries := make(map[string]*lib.Entry) + var err error + + switch { + case strings.HasPrefix(g.URI, "http://"), strings.HasPrefix(g.URI, "https://"): + err = g.walkRemoteFile(g.URI, entries) + default: + err = g.walkLocalFile(g.URI, entries) + } + + if err != nil { + return nil, err + } + + if len(entries) == 0 { + return nil, fmt.Errorf("❌ [type %s | action %s] no entry is newly generated", typeGeoIPdatIn, g.Action) + } + + var ignoreIPType lib.IgnoreIPOption + switch g.OnlyIPType { + case lib.IPv4: + ignoreIPType = lib.IgnoreIPv6 + case lib.IPv6: + ignoreIPType = lib.IgnoreIPv4 + } + + // Filter want list + wantList := make(map[string]bool) + for _, want := range g.Want { + if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { + wantList[want] = true + } + } + + for _, entry := range entries { + name := entry.GetName() + if len(wantList) > 0 && !wantList[name] { + continue + } + + switch g.Action { + case lib.ActionAdd: + if err := container.Add(entry, ignoreIPType); err != nil { + return nil, err + } + case lib.ActionRemove: + container.Remove(name, ignoreIPType) + } + } + + return container, nil +} + +func (g *geoIPDatIn) walkLocalFile(path string, entries map[string]*lib.Entry) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + if err := g.generateEntries(file, entries); err != nil { + return err + } + + return nil +} + +func (g *geoIPDatIn) walkRemoteFile(url string, entries map[string]*lib.Entry) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("failed to get remote file %s, http status code %d", url, resp.StatusCode) + } + + if err := g.generateEntries(resp.Body, entries); err != nil { + return err + } + + return nil +} + +func (g *geoIPDatIn) generateEntries(reader io.Reader, entries map[string]*lib.Entry) error { + geoipBytes, err := io.ReadAll(reader) + if err != nil { + return err + } + + var geoipList router.GeoIPList + if err := proto.Unmarshal(geoipBytes, &geoipList); err != nil { + return err + } + + for _, geoip := range geoipList.Entry { + var entry *lib.Entry + name := geoip.CountryCode + if theEntry, found := entries[name]; found { + fmt.Printf("⚠️ [type %s | action %s] found duplicated entry: %s. Process anyway\n", typeGeoIPdatIn, g.Action, name) + entry = theEntry + } else { + entry = lib.NewEntry(name) + } + + for _, v2rayCIDR := range geoip.Cidr { + ipStr := net.IP(v2rayCIDR.GetIp()).String() + "/" + fmt.Sprint(v2rayCIDR.GetPrefix()) + switch g.Action { + case lib.ActionAdd: + if err := entry.AddPrefix(ipStr); err != nil { + return err + } + case lib.ActionRemove: + if err := entry.RemovePrefix(ipStr); err != nil { + return err + } + } + } + + entries[name] = entry + } + + return nil +} diff --git a/plugin/v2ray/dat_out.go b/plugin/v2ray/dat_out.go new file mode 100644 index 00000000..7e2f01ec --- /dev/null +++ b/plugin/v2ray/dat_out.go @@ -0,0 +1,227 @@ +package v2ray + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/Loyalsoldier/geoip/lib" + "github.com/v2fly/v2ray-core/v4/app/router" + "github.com/v2fly/v2ray-core/v4/infra/conf/rule" + "google.golang.org/protobuf/proto" +) + +const ( + typeGeoIPdatOut = "v2rayGeoIPDat" + descGeoIPdatOut = "Convert data to V2Ray GeoIP dat format" +) + +var ( + defaultOutputName = "geoip.dat" + defaultOutputDir = filepath.Join("./", "output", "dat") +) + +func init() { + lib.RegisterOutputConfigCreator(typeGeoIPdatOut, func(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { + return newGeoIPDat(action, data) + }) + lib.RegisterOutputConverter(typeGeoIPdatOut, &geoIPDatOut{ + Description: descGeoIPdatOut, + }) +} + +func newGeoIPDat(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { + var tmp struct { + OutputName string `json:"outputName"` + OutputDir string `json:"outputDir"` + Want []string `json:"wantedList"` + OneFilePerList bool `json:"oneFilePerList"` + OnlyIPType lib.IPType `json:"onlyIPType"` + } + + if len(data) > 0 { + if err := json.Unmarshal(data, &tmp); err != nil { + return nil, err + } + } + + if tmp.OutputName == "" { + tmp.OutputName = defaultOutputName + } + + if tmp.OutputDir == "" { + tmp.OutputDir = defaultOutputDir + } + + return &geoIPDatOut{ + Type: typeGeoIPdatOut, + Action: action, + Description: descGeoIPdatOut, + OutputName: tmp.OutputName, + OutputDir: tmp.OutputDir, + Want: tmp.Want, + OneFilePerList: tmp.OneFilePerList, + OnlyIPType: tmp.OnlyIPType, + }, nil +} + +type geoIPDatOut struct { + Type string + Action lib.Action + Description string + OutputName string + OutputDir string + Want []string + OneFilePerList bool + OnlyIPType lib.IPType +} + +func (g *geoIPDatOut) GetType() string { + return g.Type +} + +func (g *geoIPDatOut) GetAction() lib.Action { + return g.Action +} + +func (g *geoIPDatOut) GetDescription() string { + return g.Description +} + +func (g *geoIPDatOut) Output(container lib.Container) error { + // Filter want list + wantList := make(map[string]bool) + for _, want := range g.Want { + if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { + wantList[want] = true + } + } + + geoIPList := new(router.GeoIPList) + geoIPList.Entry = make([]*router.GeoIP, 0, 300) + updated := false + switch len(wantList) { + case 0: + for entry := range container.Loop() { + geoIP, err := g.generateGeoIP(entry) + if err != nil { + return err + } + geoIPList.Entry = append(geoIPList.Entry, geoIP) + updated = true + + if g.OneFilePerList { + geoIPBytes, err := proto.Marshal(geoIPList) + if err != nil { + return err + } + filename := strings.ToLower(entry.GetName()) + ".dat" + if err := g.writeFile(filename, geoIPBytes); err != nil { + return err + } + geoIPList.Entry = nil + } + } + + default: + for name := range wantList { + entry, found := container.GetEntry(name) + if !found { + log.Printf("❌ entry %s not found", name) + continue + } + geoIP, err := g.generateGeoIP(entry) + if err != nil { + return err + } + geoIPList.Entry = append(geoIPList.Entry, geoIP) + updated = true + + if g.OneFilePerList { + geoIPBytes, err := proto.Marshal(geoIPList) + if err != nil { + return err + } + filename := strings.ToLower(entry.GetName()) + ".dat" + if err := g.writeFile(filename, geoIPBytes); err != nil { + return err + } + geoIPList.Entry = nil + } + } + } + + // Sort to make reproducible builds + g.sort(geoIPList) + + if !g.OneFilePerList && updated { + geoIPBytes, err := proto.Marshal(geoIPList) + if err != nil { + return err + } + if err := g.writeFile(g.OutputName, geoIPBytes); err != nil { + return err + } + } + + return nil +} + +func (g *geoIPDatOut) generateGeoIP(entry *lib.Entry) (*router.GeoIP, error) { + var entryCidr []string + var err error + switch g.OnlyIPType { + case lib.IPv4: + entryCidr, err = entry.MarshalText(lib.IgnoreIPv6) + case lib.IPv6: + entryCidr, err = entry.MarshalText(lib.IgnoreIPv4) + default: + entryCidr, err = entry.MarshalText() + } + if err != nil { + return nil, err + } + + v2rayCIDR := make([]*router.CIDR, 0, 1024) + for _, cidrStr := range entryCidr { + cidr, err := rule.ParseIP(cidrStr) + if err != nil { + return nil, err + } + v2rayCIDR = append(v2rayCIDR, cidr) + } + + if len(v2rayCIDR) > 0 { + return &router.GeoIP{ + CountryCode: entry.GetName(), + Cidr: v2rayCIDR, + }, nil + } + + return nil, fmt.Errorf("entry %s has no CIDR", entry.GetName()) +} + +// Sort by country code to make reproducible builds +func (g *geoIPDatOut) sort(list *router.GeoIPList) { + sort.SliceStable(list.Entry, func(i, j int) bool { + return list.Entry[i].CountryCode < list.Entry[j].CountryCode + }) +} + +func (g *geoIPDatOut) writeFile(filename string, geoIPBytes []byte) error { + if err := os.MkdirAll(g.OutputDir, 0755); err != nil { + return err + } + + if err := os.WriteFile(filepath.Join(g.OutputDir, filename), geoIPBytes, 0644); err != nil { + return err + } + + log.Printf("✅ [%s] %s --> %s", g.Type, filename, g.OutputDir) + + return nil +} -- cgit v1.3.1