Initial commit

redpilllabs · Jan 3, 2024 · 44803a5 · 44803a5
commit 44803a5
Show file tree

Hide file tree

Showing 23 changed files with 95,518 additions and 0 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -0,0 +1,160 @@
+name: Build and release GeoIP and GeoSite databases
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 23 * *"
+  push:
+    paths:
+      - 'data/community/**'
+      - 'data/ito/**'
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup environment
+        shell: bash
+        run: |
+          echo "TAG_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV
+          echo "RELEASE_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV
+
+
+      - name: Checkout codebase
+        uses: actions/checkout@v4
+
+
+      - name: Checkout v2fly/geoip repository
+        uses: actions/checkout@v4
+        with:
+          repository: v2fly/geoip
+          path: v2fly-geoip
+
+
+      - name: Checkout v2fly/geosite repository
+        uses: actions/checkout@v4
+        with:
+          repository: v2fly/domain-list-community
+          path: v2fly-geosite
+
+
+      - name: Checkout kyochikuto/sing-geoip repository
+        uses: actions/checkout@v4
+        with:
+          repository: kyochikuto/sing-geoip
+          path: sing-geoip
+
+
+      # TODO: Change to upstream when (https://github.com/SagerNet/sing-geosite/pull/9) is merged
+      - name: Checkout kyochikuto/sing-geosite repository
+        uses: actions/checkout@v4
+        with:
+          repository: kyochikuto/sing-geosite
+          path: sing-geosite
+
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: ${{ github.workspace }}/v2fly-geoip/go.mod
+
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+
+      - name: Get MaxMind GeoLite2 database
+        working-directory: ${{ github.workspace }}
+        env:
+          LICENSE_KEY: ${{ secrets.MAXMIND_LICENSE_KEY }}
+        run: |
+          curl -sSL --progress-bar "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country-CSV&license_key=${LICENSE_KEY}&suffix=zip" -o GeoLite2-Country-CSV.zip
+          curl -sSL --progress-bar "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${LICENSE_KEY}&suffix=tar.gz" -o GeoLite2-Country.tar.gz
+          curl -sSL --progress-bar "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN-CSV&license_key=${LICENSE_KEY}&suffix=zip" -o GeoLite2-ASN-CSV.zip
+          tar -xvf GeoLite2-Country.tar.gz --wildcards --strip-components=1 '*/GeoLite2-Country.mmdb'
+          mv GeoLite2-Country.mmdb ${{ github.workspace }}/data/geolite2/GeoLite2-Country.mmdb
+          unzip -j -o GeoLite2-Country-CSV.zip -d ${{ github.workspace }}/data/geolite2
+          unzip -j -o GeoLite2-ASN-CSV.zip -d ${{ github.workspace }}/data/geolite2
+          rm -f GeoLite2-Country-CSV.zip
+          rm -f GeoLite2-Country.tar.gz
+          rm -f GeoLite2-ASN-CSV.zip
+
+      - name: Get DB-IP database
+        run: |
+          curl -sSL "https://download.db-ip.com/free/dbip-country-lite-2023-12.csv.gz" -o dbip-country-lite.csv.gz
+          gunzip dbip-country-lite.csv.gz
+          rm -f dbip-country-lite.csv.gz
+          find . -type f -name "dbip*.csv" -exec mv {} ${{ github.workspace }}/data/dbip \;
+
+
+      - name: Aggregate and create IP text databases
+        run: |
+          mkdir ${{ github.workspace }}/build
+          python ./main.py
+
+
+      - name: Build Xray/v2ray GeoIP dat files
+        run: |
+          cd ${{ github.workspace }}/v2fly-geoip
+          go mod download
+          go run ./ -c ${{ github.workspace }}/data/community/v2ray/geoip/config.json
+          mv ${{ github.workspace }}/v2fly-geoip/output/dat/*.dat ${{ github.workspace }}/build
+
+
+      - name: Build Sing-Box GeoIP db file
+        run: |
+          cd ${{ github.workspace }}/sing-geoip
+          mkdir output
+          go mod download
+          go run main.go ${{ github.workspace }}/data/geolite2/GeoLite2-Country.mmdb output/geoip.db ir
+          mv output/geoip.db ${{ github.workspace }}/build/geoip.db
+
+
+      - name: Build Xray/v2ray GeoSite dat files
+        run: |
+          cd ${{ github.workspace }}/v2fly-geosite
+          go mod download
+          rm ./data/category-ir
+          rm ./data/youtube
+          cp ${{ github.workspace }}/data/community/v2ray/geosite/* ./data/
+          go run ./ --outputdir=${{ github.workspace }}/build --exportlists=category-ads-all,category-porn,ir,embargo,github,cloudflare,youtube,twitter
+          mv ${{ github.workspace }}/build/dlc.dat ${{ github.workspace }}/build/geosite.dat
+
+
+      - name: Build Sing-Box GeoSite db file
+        run: |
+          cd ${{ github.workspace }}/sing-geosite
+          mkdir output
+          go mod download
+          go run main.go ${{ github.workspace }}/build/geosite.dat output/geosite.db
+          mv output/geosite.db ${{ github.workspace }}/build/geosite.db
+
+
+      - name: Generate sha256 checksum for assets
+        run: |
+          cd ${{ github.workspace }}/build || exit 1
+          for name in $(ls *.dat *.db); do
+            sha256sum ${name} > ./${name}.sha256sum
+          done
+
+      - name: Release and upload assets
+        run: |
+          gh release create --notes "Updated database" ${{ env.TAG_NAME }} --title ${{ env.RELEASE_NAME }} \
+            ./build/agg_cidrs.csv \
+            ./build/geosite.dat \
+            ./build/geosite.dat.sha256sum \
+            ./build/geosite.db \
+            ./build/geosite.db.sha256sum \
+            ./build/geoip.dat \
+            ./build/geoip.dat.sha256sum \
+            ./build/geoip.db \
+            ./build/geoip.db.sha256sum \
+            ./build/geoip-lite.dat \
+            ./build/geoip-lite.dat.sha256sum \
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,5 @@
+/.venv
+/.idea
+/.vscode
+__pycache__
+/build
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Red Pill Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
@@ -0,0 +1,95 @@
+# Introduction
+
+This is a collaborative effort to gather an aggregated database of GeoIPs registered in Iran, China and also common IPs belonging to widely-used services. This helps users to implement ACLs on their VPN clients or cloud servers to aid in routing decisions, content blocking, and probing protection.
+
+The repository offers two types of datasets targeting different use-cases, the `agg_cidrs.csv` dataset is intended for utilization on cloud servers (`xtables` module on Linux) and is currently used in `Rainb0w` proxy installers available on [RedPillLabs](https://github.com/redpilllabs) repositories page, while `.dat` datasets are intended for use on v2ray/xray and `.db` datasets are intended for sing-box compatible clients.
+
+The `agg_cidrs.csv` dataset currently offers the following networks: `[IR, CN, CF (Cloudflare)]`.
+
+The `geoip.dat` dataset currently offers the following networks: `[ir, cloudflare, google, amazon, microsoft, github, facebook, twitter, telegram]` while the `geoip.db` only offers country tags available in GeoLite2 database.
+
+The `geosite.dat` and `geosite.db` datasets currently offer the following networks:
+
+```
+- category-ads-all -> Aggregated list of domains used for advertising
+- category-porn -> Aggregated list of domains hosting NSFW content
+- ir -> Aggregated list of Iranian domains plus regex rule for the [.ir] ccTLD
+- embargo -> Websites that have banned Iranian IPs (403 error)
+- github -> Domains belonging to GitHub
+- cloudflare -> Domains belonging to Cloudflare
+- youtube -> Domains belonging to YouTube
+- twitter -> Domains belonging to Twitter
+```
+
+# Download
+
+## Xray/v2ray core
+
+`GeoIP` [https://github.com/redpilllabs/GFIGeoIP/releases/latest/download/geoip.dat](https://github.com/redpilllabs/GFIGeoIP/releases/latest/download/geoip.dat)
+
+`GeoSite` [https://github.com/redpilllabs/GFIGeoIP/releases/latest/download/geosite.dat](https://github.com/redpilllabs/GFIGeoIP/releases/latest/download/geosite.dat)
+
+## Sing-Box core
+
+`GeoIP` [https://github.com/redpilllabs/GFIGeoIP/releases/latest/download/geoip.db](https://github.com/redpilllabs/GFIGeoIP/releases/latest/download/geoip.db)
+
+`GeoSite` [https://github.com/redpilllabs/GFIGeoIP/releases/latest/download/geosite.db](https://github.com/redpilllabs/GFIGeoIP/releases/latest/download/geosite.db)
+
+# How do I use it?
+
+The following is a lean example of v2ray/xray client configuration:
+
+```
+"outbounds": [
+  {
+    "tag": "direct",
+    "protocol": "freedom",
+    "settings": {}
+  },
+  {
+    "tag": "block",
+    "protocol": "blackhole",
+    "settings": {}
+  }
+],
+"routing": {
+  "domainStrategy": "IPIfNonMatch",
+  "rules": [
+    {
+      "outboundTag": "block",
+      "domain": [
+        "geosite:category-ads-all"
+      ],
+      "type": "field"
+    },
+    {
+      "outboundTag": "direct",
+      "ip": [
+        "geoip:private",
+        "geoip:ir",
+        "geosite:ir"
+      ],
+      "type": "field"
+    }
+  ]
+}
+```
+
+# Contribution
+
+Entries in the `embargo` database should be tested with the following curl command and return with a 403 HTTP status code :
+
+```
+curl -I -s -o /dev/null -w "%{http_code}" https://DOMAIN.TLD
+```
+
+# What sources are used for aggregation?
+
+Data are pulled from multiple sources such as DBIP, MaxMind Geolite2, ITO, and manually inspected networks.
+
+# Credits
+
+- All maintainers and contributors to the project.
+- [MaxMind GeoLite2 ®](https://maxmind.com)
+- [DB-IP](https://db-ip.com)
+- [Project V](https://github.com/v2fly)
diff --git a/data/community/ipv4_IR.csv b/data/community/ipv4_IR.csv
@@ -0,0 +1,4 @@
+Network,Tag
+46.209.0.0/16,IR
+5.160.0.0/16,IR
+81.12.0.0/17,IR
diff --git a/data/community/ipv4_XX.csv b/data/community/ipv4_XX.csv
@@ -0,0 +1,29 @@
+Network,Tag
+204.89.193.0/24,XX
+204.93.185.0/24,XX
+208.99.64.0/19,XX
+209.239.160.0/20,XX
+216.18.160.0/19,XX
+64.202.125.0/24,XX
+64.202.126.0/24,XX
+64.210.128.0/19,XX
+64.210.148.0/24,XX
+64.88.240.0/24,XX
+64.88.244.0/24,XX
+66.254.105.0/24,XX
+66.254.114.0/24,XX
+66.254.122.0/24,XX
+66.254.126.0/24,XX
+66.254.127.0/24,XX
+66.254.96.0/19,XX
+67.22.48.0/24,XX
+67.22.50.0/24,XX
+67.22.51.0/24,XX
+67.22.52.0/24,XX
+67.22.53.0/24,XX
+67.22.54.0/24,XX
+67.22.55.0/24,XX
+67.22.56.0/24,XX
+67.22.57.0/24,XX
+67.22.58.0/24,XX
+67.22.59.0/24,XX
diff --git a/data/community/ipv6_XX.csv b/data/community/ipv6_XX.csv
@@ -0,0 +1,3 @@
+Network,Tag
+2001:18b8::/32,XX
+2001:18b8:d45::/48,XX