Feat: fetch cidr data from ripe.net & add geoip:netflix

4 files changed, 49 insertions, 4 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 078e7e88..eddea8d4 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -21,21 +21,37 @@ jobs:
 
       - name: Set variables
         run: |
+          echo "PATH=$(go env GOPATH)/bin:${PATH}" >> $GITHUB_ENV
           echo "TAG_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV
           echo "RELEASE_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV
         shell: bash
 
+      - name: Install cidr-merger
+        run: |
+          go get github.com/zhanhb/cidr-merger
+
       - name: Checkout codebase
         uses: actions/[email protected]
         with:
           fetch-depth: 0
 
-      - name: Update appended CIDR lists automatically
+      - name: Fetch lists from ripe.net
+        run: |
+          chmod +x asn.sh
+          ./asn.sh
+
+      - name: Append more CIDRs
         run: |
-          curl -sSL https://www.cloudflare.com/ips-v4 | grep "/" > data/cloudflare
+          curl -sSL https://www.cloudflare.com/ips-v4 | grep "/" >> data/cloudflare
           curl -sSL https://www.cloudflare.com/ips-v6 | grep "/" >> data/cloudflare
-          curl -sSL https://api.fastly.com/public-ip-list | jq --raw-output '.addresses[],.ipv6_addresses[]' > data/fastly
-          curl -sSL https://d7uri8nf7uskq.cloudfront.net/tools/list-cloudfront-ips | jq --raw-output '.CLOUDFRONT_GLOBAL_IP_LIST[],.CLOUDFRONT_REGIONAL_EDGE_IP_LIST[]' > data/cloudfront
+          curl -sSL https://api.fastly.com/public-ip-list | jq --raw-output '.addresses[],.ipv6_addresses[]' >> data/fastly
+          curl -sSL https://d7uri8nf7uskq.cloudfront.net/tools/list-cloudfront-ips | jq --raw-output '.CLOUDFRONT_GLOBAL_IP_LIST[],.CLOUDFRONT_REGIONAL_EDGE_IP_LIST[]' >> data/cloudfront
+
+      - name: Merge CIDR in files except ./data/private
+        run: |
+          for file in $(find ./data -type f ! -name "private"); do
+            cidr-merger --batch -s -o ${file} ${file}
+          done
 
       - name: Commit file changes
         run: |
diff --git a/README.md b/README.md
index 1461839b..b75ee552 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,7 @@
   - `geoip:cloudflare`
   - `geoip:cloudfront`
   - `geoip:fastly`
+  - `geoip:netflix`
   - `geoip:telegram`
 
 ## 说明
diff --git a/asn.csv b/asn.csv
new file mode 100644
index 00000000..11c594c9
--- /dev/null
+++ b/asn.csv
@@ -0,0 +1,4 @@
+cloudflare,AS395747|AS394536|AS209242|AS203898|AS202623|AS14789|AS139242|AS133877|AS13335|AS132892
+fastly,AS54113|AS394192
+netflix,AS55095|AS40027|AS394406|AS2906
+telegram,AS62041|AS62014|AS59930|AS44907|AS211157
diff --git a/asn.sh b/asn.sh
new file mode 100755
index 00000000..81c72c18
--- /dev/null
+++ b/asn.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+input="./asn.csv"
+mkdir -p ./tmp
+
+while IFS= read -r line; do
+  filename=$(echo ${line} | awk -F ',' '{print $1}')
+  IFS='|' read -r -a asns <<<$(echo ${line} | awk -F ',' '{print $2}')
+  file="data/${filename}"
+
+  echo "==================================="
+  echo "Generating ${filename} CIDR list..."
+  rm -rf ${file} && touch ${file}
+  for asn in ${asns[@]}; do
+    url="https://stat.ripe.net/data/ris-prefixes/data.json?list_prefixes=true&types=o&resource=${asn}"
+    echo "-----------------------"
+    echo "Fetching ${asn}..."
+    echo "# ${asn}" >>${file}
+    curl -sL ${url} -o ./tmp/${filename}-${asn}.txt \
+      -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'
+    jq --raw-output '.data.prefixes.v4.originating[]' ./tmp/${filename}-${asn}.txt | sort -u >>${file}
+    jq --raw-output '.data.prefixes.v6.originating[]' ./tmp/${filename}-${asn}.txt | sort -u >>${file}
+  done
+done <${input}