Refactor: use plugin architecture to support multiple I/O formats

8 files changed, 680 insertions, 0 deletions

diff --git a/plugin/plaintext/clash_in.go b/plugin/plaintext/clash_in.go
new file mode 100644
index 00000000..7cf36338
--- /dev/null
+++ b/plugin/plaintext/clash_in.go
@@ -0,0 +1,36 @@
+package plaintext
+
+import (
+	"encoding/json"
+
+	"github.com/Loyalsoldier/geoip/lib"
+)
+
+/*
+The types in this file extend the type `typeTextIn`,
+which make it possible to support more formats for the project.
+*/
+
+const (
+	typeClashRuleSetClassicalIn = "clashRuleSetClassical"
+	descClashClassicalIn        = "Convert classical type of Clash RuleSet to other formats (just processing IP & CIDR lines)"
+
+	typeClashRuleSetIPCIDRIn = "clashRuleSet"
+	descClashRuleSetIn       = "Convert ipcidr type of Clash RuleSet to other formats"
+)
+
+func init() {
+	lib.RegisterInputConfigCreator(typeClashRuleSetClassicalIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) {
+		return newTextIn(typeClashRuleSetClassicalIn, action, data)
+	})
+	lib.RegisterInputConverter(typeClashRuleSetClassicalIn, &textIn{
+		Description: descClashClassicalIn,
+	})
+
+	lib.RegisterInputConfigCreator(typeClashRuleSetIPCIDRIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) {
+		return newTextIn(typeClashRuleSetIPCIDRIn, action, data)
+	})
+	lib.RegisterInputConverter(typeClashRuleSetIPCIDRIn, &textIn{
+		Description: descClashRuleSetIn,
+	})
+}
diff --git a/plugin/plaintext/clash_out.go b/plugin/plaintext/clash_out.go
new file mode 100644
index 00000000..a7feab75
--- /dev/null
+++ b/plugin/plaintext/clash_out.go
@@ -0,0 +1,36 @@
+package plaintext
+
+import (
+	"encoding/json"
+
+	"github.com/Loyalsoldier/geoip/lib"
+)
+
+/*
+The types in this file extend the type `typeTextOut`,
+which make it possible to support more formats for the project.
+*/
+
+const (
+	typeClashRuleSetClassicalOut = "clashRuleSetClassical"
+	descClashClassicalOut        = "Convert data to classical type of Clash RuleSet"
+
+	typeClashRuleSetIPCIDROut = "clashRuleSet"
+	descClashRuleSetOut       = "Convert data to ipcidr type of Clash RuleSet"
+)
+
+func init() {
+	lib.RegisterOutputConfigCreator(typeClashRuleSetClassicalOut, func(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) {
+		return newTextOut(typeClashRuleSetClassicalOut, action, data)
+	})
+	lib.RegisterOutputConverter(typeClashRuleSetClassicalOut, &textOut{
+		Description: descClashClassicalOut,
+	})
+
+	lib.RegisterOutputConfigCreator(typeClashRuleSetIPCIDROut, func(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) {
+		return newTextOut(typeClashRuleSetIPCIDROut, action, data)
+	})
+	lib.RegisterOutputConverter(typeClashRuleSetIPCIDROut, &textOut{
+		Description: descClashRuleSetOut,
+	})
+}
diff --git a/plugin/plaintext/common_in.go b/plugin/plaintext/common_in.go
new file mode 100644
index 00000000..c8bfe367
--- /dev/null
+++ b/plugin/plaintext/common_in.go
@@ -0,0 +1,106 @@
+package plaintext
+
+import (
+	"bufio"
+	"io"
+	"strings"
+
+	"github.com/Loyalsoldier/geoip/lib"
+	"gopkg.in/yaml.v2"
+)
+
+type textIn struct {
+	Type        string
+	Action      lib.Action
+	Description string
+	Name        string
+	URI         string
+	InputDir    string
+	OnlyIPType  lib.IPType
+}
+
+func (t *textIn) scanFile(reader io.Reader, entry *lib.Entry) error {
+	var err error
+	switch t.Type {
+	case typeTextIn:
+		err = t.scanFileForTextIn(reader, entry)
+	case typeClashRuleSetClassicalIn:
+		err = t.scanFileForClashClassicalRuleSetInAndSurgeIn(reader, entry)
+	case typeClashRuleSetIPCIDRIn:
+		err = t.scanFileForClashRuleSetIn(reader, entry)
+	case typeSurgeRuleSetIn:
+		err = t.scanFileForClashClassicalRuleSetInAndSurgeIn(reader, entry)
+	default:
+		return lib.ErrNotSupportedFormat
+	}
+
+	return err
+}
+
+func (t *textIn) scanFileForTextIn(reader io.Reader, entry *lib.Entry) error {
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" {
+			continue
+		}
+		if err := entry.AddPrefix(line); err != nil {
+			return err
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (t *textIn) scanFileForClashRuleSetIn(reader io.Reader, entry *lib.Entry) error {
+	var payload struct {
+		Payload []string `yaml:"payload"`
+	}
+
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return err
+	}
+
+	if err := yaml.Unmarshal(data, &payload); err != nil {
+		return err
+	}
+
+	for _, cidrStr := range payload.Payload {
+		if err := entry.AddPrefix(strings.TrimSpace(cidrStr)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (t *textIn) scanFileForClashClassicalRuleSetInAndSurgeIn(reader io.Reader, entry *lib.Entry) error {
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		line := strings.ToLower(strings.TrimSpace(scanner.Text()))
+		if line == "" {
+			continue
+		}
+
+		switch {
+		case strings.HasPrefix(line, "ip-cidr,"), strings.HasPrefix(line, "ip-cidr6,"):
+			parts := strings.Split(line, ",")
+			if len(parts) > 1 {
+				if err := entry.AddPrefix(strings.TrimSpace(parts[1])); err != nil {
+					return err
+				}
+			}
+		default:
+			continue
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/plugin/plaintext/common_out.go b/plugin/plaintext/common_out.go
new file mode 100644
index 00000000..ff4ab63b
--- /dev/null
+++ b/plugin/plaintext/common_out.go
@@ -0,0 +1,170 @@
+package plaintext
+
+import (
+	"bytes"
+	"encoding/json"
+	"log"
+	"net"
+	"os"
+	"path/filepath"
+
+	"github.com/Loyalsoldier/geoip/lib"
+)
+
+var (
+	defaultOutputDirForTextOut                  = filepath.Join("./", "output", "text")
+	defaultOutputDirForClashRuleSetClassicalOut = filepath.Join("./", "output", "clash", "classical")
+	defaultOutputDirForClashRuleSetIPCIDROut    = filepath.Join("./", "output", "clash", "ipcidr")
+	defaultOutputDirForSurgeRuleSetOut          = filepath.Join("./", "output", "surge")
+)
+
+type textOut struct {
+	Type        string
+	Action      lib.Action
+	Description string
+	OutputDir   string
+	Want        []string
+	OnlyIPType  lib.IPType
+}
+
+func newTextOut(iType string, action lib.Action, data json.RawMessage) (lib.OutputConverter, error) {
+	var tmp struct {
+		OutputDir  string     `json:"outputDir"`
+		Want       []string   `json:"wantedList"`
+		OnlyIPType lib.IPType `json:"onlyIPType"`
+	}
+
+	if len(data) > 0 {
+		if err := json.Unmarshal(data, &tmp); err != nil {
+			return nil, err
+		}
+	}
+
+	if tmp.OutputDir == "" {
+		switch iType {
+		case typeTextOut:
+			tmp.OutputDir = defaultOutputDirForTextOut
+		case typeClashRuleSetClassicalOut:
+			tmp.OutputDir = defaultOutputDirForClashRuleSetClassicalOut
+		case typeClashRuleSetIPCIDROut:
+			tmp.OutputDir = defaultOutputDirForClashRuleSetIPCIDROut
+		case typeSurgeRuleSetOut:
+			tmp.OutputDir = defaultOutputDirForSurgeRuleSetOut
+		}
+	}
+
+	return &textOut{
+		Type:        iType,
+		Action:      action,
+		Description: descTextOut,
+		OutputDir:   tmp.OutputDir,
+		Want:        tmp.Want,
+		OnlyIPType:  tmp.OnlyIPType,
+	}, nil
+}
+
+func (t *textOut) marshalBytes(entry *lib.Entry) ([]byte, error) {
+	var err error
+
+	var entryCidr []string
+	switch t.OnlyIPType {
+	case lib.IPv4:
+		entryCidr, err = entry.MarshalText(lib.IgnoreIPv6)
+	case lib.IPv6:
+		entryCidr, err = entry.MarshalText(lib.IgnoreIPv4)
+	default:
+		entryCidr, err = entry.MarshalText()
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var buf bytes.Buffer
+	switch t.Type {
+	case typeTextOut:
+		err = t.marshalBytesForTextOut(&buf, entryCidr)
+	case typeClashRuleSetClassicalOut:
+		err = t.marshalBytesForClashRuleSetClassicalOut(&buf, entryCidr)
+	case typeClashRuleSetIPCIDROut:
+		err = t.marshalBytesForClashRuleSetIPCIDROut(&buf, entryCidr)
+	case typeSurgeRuleSetOut:
+		err = t.marshalBytesForSurgeRuleSetOut(&buf, entryCidr)
+	default:
+		return nil, lib.ErrNotSupportedFormat
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func (t *textOut) marshalBytesForTextOut(buf *bytes.Buffer, entryCidr []string) error {
+	for _, cidr := range entryCidr {
+		buf.WriteString(cidr)
+		buf.WriteString("\n")
+	}
+	return nil
+}
+
+func (t *textOut) marshalBytesForClashRuleSetClassicalOut(buf *bytes.Buffer, entryCidr []string) error {
+	buf.WriteString("payload:\n")
+	for _, cidr := range entryCidr {
+		ip, _, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return err
+		}
+		if ip.To4() != nil {
+			buf.WriteString("  - IP-CIDR,")
+		} else {
+			buf.WriteString("  - IP-CIDR6,")
+		}
+		buf.WriteString(cidr)
+		buf.WriteString("\n")
+	}
+
+	return nil
+}
+
+func (t *textOut) marshalBytesForClashRuleSetIPCIDROut(buf *bytes.Buffer, entryCidr []string) error {
+	buf.WriteString("payload:\n")
+	for _, cidr := range entryCidr {
+		buf.WriteString("  - '")
+		buf.WriteString(cidr)
+		buf.WriteString("'\n")
+	}
+
+	return nil
+}
+
+func (t *textOut) marshalBytesForSurgeRuleSetOut(buf *bytes.Buffer, entryCidr []string) error {
+	for _, cidr := range entryCidr {
+		ip, _, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return err
+		}
+		if ip.To4() != nil {
+			buf.WriteString("IP-CIDR,")
+		} else {
+			buf.WriteString("IP-CIDR6,")
+		}
+		buf.WriteString(cidr)
+		buf.WriteString("\n")
+	}
+
+	return nil
+}
+
+func (t *textOut) writeFile(filename string, data []byte) error {
+	if err := os.MkdirAll(t.OutputDir, 0755); err != nil {
+		return err
+	}
+
+	if err := os.WriteFile(filepath.Join(t.OutputDir, filename), data, 0644); err != nil {
+		return err
+	}
+
+	log.Printf("✅ [%s] %s --> %s", t.Type, filename, t.OutputDir)
+
+	return nil
+}
diff --git a/plugin/plaintext/surge_in.go b/plugin/plaintext/surge_in.go
new file mode 100644
index 00000000..d409a782
--- /dev/null
+++ b/plugin/plaintext/surge_in.go
@@ -0,0 +1,26 @@
+package plaintext
+
+import (
+	"encoding/json"
+
+	"github.com/Loyalsoldier/geoip/lib"
+)
+
+/*
+The types in this file extend the type `typeTextIn`,
+which make it possible to support more formats for the project.
+*/
+
+const (
+	typeSurgeRuleSetIn = "surgeRuleSet"
+	descSurgeRuleSetIn = "Convert Surge RuleSet to other formats (just processing IP & CIDR lines)"
+)
+
+func init() {
+	lib.RegisterInputConfigCreator(typeSurgeRuleSetIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) {
+		return newTextIn(typeSurgeRuleSetIn, action, data)
+	})
+	lib.RegisterInputConverter(typeSurgeRuleSetIn, &textIn{
+		Description: descSurgeRuleSetIn,
+	})
+}
diff --git a/plugin/plaintext/surge_out.go b/plugin/plaintext/surge_out.go
new file mode 100644
index 00000000..c3868423
--- /dev/null
+++ b/plugin/plaintext/surge_out.go
@@ -0,0 +1,26 @@
+package plaintext
+
+import (
+	"encoding/json"
+
+	"github.com/Loyalsoldier/geoip/lib"
+)
+
+/*
+The types in this file extend the type `typeTextOut`,
+which make it possible to support more formats for the project.
+*/
+
+const (
+	typeSurgeRuleSetOut = "surgeRuleSet"
+	descSurgeRuleSetOut = "Convert data to Surge RuleSet"
+)
+
+func init() {
+	lib.RegisterOutputConfigCreator(typeSurgeRuleSetOut, func(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) {
+		return newTextOut(typeSurgeRuleSetOut, action, data)
+	})
+	lib.RegisterOutputConverter(typeSurgeRuleSetOut, &textOut{
+		Description: descSurgeRuleSetOut,
+	})
+}
diff --git a/plugin/plaintext/text_in.go b/plugin/plaintext/text_in.go
new file mode 100644
index 00000000..64ad3ad3
--- /dev/null
+++ b/plugin/plaintext/text_in.go
@@ -0,0 +1,202 @@
+package plaintext
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/Loyalsoldier/geoip/lib"
+)
+
+const (
+	typeTextIn = "text"
+	descTextIn = "Convert plaintext IP & CIDR to other formats"
+)
+
+func init() {
+	lib.RegisterInputConfigCreator(typeTextIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) {
+		return newTextIn(typeTextIn, action, data)
+	})
+	lib.RegisterInputConverter(typeTextIn, &textIn{
+		Description: descTextIn,
+	})
+}
+
+func newTextIn(iType string, action lib.Action, data json.RawMessage) (lib.InputConverter, error) {
+	var tmp struct {
+		Name       string     `json:"name"`
+		URI        string     `json:"uri"`
+		InputDir   string     `json:"inputDir"`
+		OnlyIPType lib.IPType `json:"onlyIPType"`
+	}
+
+	if strings.TrimSpace(iType) == "" {
+		return nil, fmt.Errorf("type is required")
+	}
+
+	if len(data) > 0 {
+		if err := json.Unmarshal(data, &tmp); err != nil {
+			return nil, err
+		}
+	}
+
+	if tmp.Name == "" && tmp.URI == "" && tmp.InputDir == "" {
+		return nil, fmt.Errorf("type %s | action %s missing inputdir or name or uri", typeTextIn, action)
+	}
+
+	if (tmp.Name != "" && tmp.URI == "") || (tmp.Name == "" && tmp.URI != "") {
+		return nil, fmt.Errorf("type %s | action %s name & uri must be specified together", typeTextIn, action)
+	}
+
+	return &textIn{
+		Type:        iType,
+		Action:      action,
+		Description: descTextIn,
+		Name:        tmp.Name,
+		URI:         tmp.URI,
+		InputDir:    tmp.InputDir,
+		OnlyIPType:  tmp.OnlyIPType,
+	}, nil
+}
+
+func (t *textIn) GetType() string {
+	return t.Type
+}
+
+func (t *textIn) GetAction() lib.Action {
+	return t.Action
+}
+
+func (t *textIn) GetDescription() string {
+	return t.Description
+}
+
+func (t *textIn) Input(container lib.Container) (lib.Container, error) {
+	entries := make(map[string]*lib.Entry)
+	var err error
+
+	switch {
+	case t.InputDir != "":
+		err = t.walkDir(t.InputDir, entries)
+	case t.Name != "" && t.URI != "":
+		switch {
+		case strings.HasPrefix(t.URI, "http://"), strings.HasPrefix(t.URI, "https://"):
+			err = t.walkRemoteFile(t.URI, t.Name, entries)
+		default:
+			err = t.walkLocalFile(t.URI, t.Name, entries)
+		}
+	default:
+		return nil, fmt.Errorf("config missing argument inputDir or name or uri")
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	var ignoreIPType lib.IgnoreIPOption
+	switch t.OnlyIPType {
+	case lib.IPv4:
+		ignoreIPType = lib.IgnoreIPv6
+	case lib.IPv6:
+		ignoreIPType = lib.IgnoreIPv4
+	}
+
+	if len(entries) == 0 {
+		return nil, fmt.Errorf("type %s | action %s no entry are generated", t.Type, t.Action)
+	}
+
+	for _, entry := range entries {
+		switch t.Action {
+		case lib.ActionAdd:
+			if err := container.Add(entry, ignoreIPType); err != nil {
+				return nil, err
+			}
+		case lib.ActionRemove:
+			container.Remove(entry.GetName(), ignoreIPType)
+		case lib.ActionReplace:
+			container.Replace(entry, ignoreIPType)
+		}
+	}
+
+	return container, nil
+}
+
+func (t *textIn) walkDir(dir string, entries map[string]*lib.Entry) error {
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		if err := t.walkLocalFile(path, "", entries); err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	return err
+}
+
+func (t *textIn) walkLocalFile(path, name string, entries map[string]*lib.Entry) error {
+	name = strings.TrimSpace(name)
+	var filename string
+	if name != "" {
+		filename = name
+	} else {
+		filename = filepath.Base(path)
+	}
+
+	// check filename
+	if !regexp.MustCompile(`^[a-zA-Z0-9_.\-]+$`).MatchString(filename) {
+		return fmt.Errorf("filename %s cannot be entry name, please remove special characters in it", filename)
+	}
+	dotIndex := strings.LastIndex(filename, ".")
+	if dotIndex > 0 {
+		filename = filename[:dotIndex]
+	}
+
+	if _, found := entries[filename]; found {
+		return fmt.Errorf("found duplicated file %s", filename)
+	}
+
+	entry := lib.NewEntry(filename)
+	file, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	if err := t.scanFile(file, entry); err != nil {
+		return err
+	}
+
+	entries[filename] = entry
+
+	return nil
+}
+
+func (t *textIn) walkRemoteFile(url, name string, entries map[string]*lib.Entry) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("failed to get remote file %s, http status code %d", url, resp.StatusCode)
+	}
+
+	entry := lib.NewEntry(name)
+	if err := t.scanFile(resp.Body, entry); err != nil {
+		return err
+	}
+
+	entries[name] = entry
+	return nil
+}
diff --git a/plugin/plaintext/text_out.go b/plugin/plaintext/text_out.go
new file mode 100644
index 00000000..7bdfaf42
--- /dev/null
+++ b/plugin/plaintext/text_out.go
@@ -0,0 +1,78 @@
+package plaintext
+
+import (
+	"encoding/json"
+	"log"
+	"strings"
+
+	"github.com/Loyalsoldier/geoip/lib"
+)
+
+const (
+	typeTextOut = "text"
+	descTextOut = "Convert data to plaintext CIDR format"
+)
+
+func init() {
+	lib.RegisterOutputConfigCreator(typeTextOut, func(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) {
+		return newTextOut(typeTextOut, action, data)
+	})
+	lib.RegisterOutputConverter(typeTextOut, &textOut{
+		Description: descTextOut,
+	})
+}
+
+func (t *textOut) GetType() string {
+	return t.Type
+}
+
+func (t *textOut) GetAction() lib.Action {
+	return t.Action
+}
+
+func (t *textOut) GetDescription() string {
+	return t.Description
+}
+
+func (t *textOut) Output(container lib.Container) error {
+	// Filter want list
+	wantList := make(map[string]bool)
+	for _, want := range t.Want {
+		if want = strings.ToUpper(strings.TrimSpace(want)); want != "" {
+			wantList[want] = true
+		}
+	}
+
+	switch len(wantList) {
+	case 0:
+		for entry := range container.Loop() {
+			data, err := t.marshalBytes(entry)
+			if err != nil {
+				return err
+			}
+			filename := strings.ToLower(entry.GetName()) + ".txt"
+			if err := t.writeFile(filename, data); err != nil {
+				return err
+			}
+		}
+
+	default:
+		for name := range wantList {
+			entry, found := container.GetEntry(name)
+			if !found {
+				log.Printf("entry %s not found", name)
+				continue
+			}
+			data, err := t.marshalBytes(entry)
+			if err != nil {
+				return err
+			}
+			filename := strings.ToLower(entry.GetName()) + ".txt"
+			if err := t.writeFile(filename, data); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}