forked from Loyalsoldier/v2ray-rules-dat
-
Notifications
You must be signed in to change notification settings - Fork 11
250 lines (227 loc) · 15.9 KB
/
run.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
name: Build V2Ray rules dat files
on:
workflow_dispatch:
schedule:
- cron: "0 22 * * *"
push:
branches:
- master
paths-ignore:
- "**/README.md"
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Setup Go 1.x.y
uses: actions/setup-go@v3
with:
go-version: ^1.18
- name: Set variables
run: |
echo "RELEASE_NAME=Released on $(date +%Y%m%d%H%M)" >> $GITHUB_ENV
echo "TAG_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV
echo "CHINA_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf" >> $GITHUB_ENV
echo "GOOGLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/google.china.conf" >> $GITHUB_ENV
echo "APPLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf" >> $GITHUB_ENV
echo "GREATFIRE_DOMAINS_URL=https://raw.githubusercontent.com/Loyalsoldier/cn-blocked-domain/release/domains.txt" >> $GITHUB_ENV
echo "EASYLISTCHINA_EASYLIST_REJECT_URL=https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt" >> $GITHUB_ENV
echo "PETERLOWE_REJECT_URL=https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext" >> $GITHUB_ENV
echo "ADGUARD_DNS_REJECT_URL=https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt" >> $GITHUB_ENV
echo "DANPOLLOCK_REJECT_URL=https://someonewhocares.org/hosts/hosts" >> $GITHUB_ENV
echo "CUSTOM_DIRECT=https://raw.githubusercontent.com/Loyalsoldier/domain-list-custom/release/cn.txt" >> $GITHUB_ENV
echo "CUSTOM_PROXY=https://raw.githubusercontent.com/Loyalsoldier/domain-list-custom/release/geolocation-!cn.txt" >> $GITHUB_ENV
echo "WIN_SPY=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt" >> $GITHUB_ENV
echo "WIN_UPDATE=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/update.txt" >> $GITHUB_ENV
echo "WIN_EXTRA=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/extra.txt" >> $GITHUB_ENV
echo "OISD_FULL=https://hosts.oisd.nl/" >> $GITHUB_ENV
echo "OISD_NSFW=https://hosts.oisd.nl/nsfw/" >> $GITHUB_ENV
echo "DOH_DNS=https://raw.githubusercontent.com/dibdot/DoH-IP-blocklists/master/doh-domains.txt" >> $GITHUB_ENV
echo "RULE_ADS=https://raw.githubusercontent.com/malikshi/v2ray-rules-dat/rule/rule-ads.txt" >> $GITHUB_ENV
echo "RULE_GAME=https://raw.githubusercontent.com/malikshi/v2ray-rules-dat/rule/rule_gaming.txt" >> $GITHUB_ENV
echo "RULE_INDO=https://raw.githubusercontent.com/malikshi/v2ray-rules-dat/rule/rule_indo.txt" >> $GITHUB_ENV
echo "RULE_PS=https://raw.githubusercontent.com/malikshi/v2ray-rules-dat/rule/rule_playstore.txt" >> $GITHUB_ENV
echo "RULE_SOSMED=https://raw.githubusercontent.com/malikshi/v2ray-rules-dat/rule/rule_sosmed.txt" >> $GITHUB_ENV
echo "RULE_STREAMING=https://raw.githubusercontent.com/malikshi/v2ray-rules-dat/rule/rule_streaming.txt" >> $GITHUB_ENV
echo "RULE_UMUM=https://raw.githubusercontent.com/malikshi/v2ray-rules-dat/rule/rule_umum.txt" >> $GITHUB_ENV
echo "RULE_IPCHECK=https://raw.githubusercontent.com/malikshi/v2ray-rules-dat/rule/rule_ipcheck.txt" >> $GITHUB_ENV
echo "RULE_VIDCONFERENCE=https://raw.githubusercontent.com/malikshi/v2ray-rules-dat/rule/rule-videoconference.txt" >> $GITHUB_ENV
echo "MALICIOUS_HOST=https://raw.githubusercontent.com/elliotwutingfeng/Inversion-DNSBL-Blocklists/main/Google_hostnames_light.txt" >> $GITHUB_ENV
shell: bash
- name: Checkout the "hidden" branch of this repo
uses: actions/checkout@v3
with:
ref: hidden
- name: Checkout Loyalsoldier/domain-list-custom
uses: actions/checkout@v3
with:
repository: Loyalsoldier/domain-list-custom
path: custom
- name: Checkout v2fly/domain-list-community
uses: actions/checkout@v3
with:
repository: v2fly/domain-list-community
path: community
- name: Checkout cokebar/gfwlist2dnsmasq
uses: actions/checkout@v3
with:
repository: cokebar/gfwlist2dnsmasq
path: gfwlist2dnsmasq
- name: Get geoip.dat relative files
run: |
wget https://github.com/Loyalsoldier/geoip/raw/release/geoip.dat
wget https://github.com/Loyalsoldier/geoip/raw/release/geoip.dat.sha256sum
- name: Generate GFWList domains
run: |
cd gfwlist2dnsmasq || exit 1
chmod +x ./gfwlist2dnsmasq.sh
./gfwlist2dnsmasq.sh -l -o ./temp-gfwlist.txt
- name: Get and add direct domains into temp-direct.txt file
run: |
curl -sSL $CHINA_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > temp-direct.txt
curl -sSL ${CUSTOM_DIRECT} | perl -ne '/^(domain):([^:]+)(\n$|:@.+)/ && print "$2\n"' >> temp-direct.txt
- name: Get and add proxy domains into temp-proxy.txt file
run: |
cat ./gfwlist2dnsmasq/temp-gfwlist.txt | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' > temp-proxy.txt
curl -sSL $GREATFIRE_DOMAINS_URL | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> temp-proxy.txt
curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt
curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt
curl -sSL ${CUSTOM_PROXY} | grep -Ev ":@cn" | perl -ne '/^(domain):([^:]+)(\n$|:@.+)/ && print "$2\n"' >> temp-proxy.txt
- name: Get and add reject domains into temp-reject.txt file
run: |
curl -sSL $EASYLISTCHINA_EASYLIST_REJECT_URL | perl -ne '/^\|\|([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})\^$/ && print "$1\n"' | perl -ne 'print if not /^[0-9]{1,3}(\.[0-9]{1,3}){3}$/' > temp-reject.txt
curl -sSL $ADGUARD_DNS_REJECT_URL | perl -ne '/^\|\|([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})\^$/ && print "$1\n"' | perl -ne 'print if not /^[0-9]{1,3}(\.[0-9]{1,3}){3}$/' >> temp-reject.txt
curl -sSL $PETERLOWE_REJECT_URL | perl -ne '/^127\.0\.0\.1\s([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})$/ && print "$1\n"' >> temp-reject.txt
curl -sSL $DANPOLLOCK_REJECT_URL | perl -ne '/^127\.0\.0\.1\s([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})/ && print "$1\n"' | sed '1d' >> temp-reject.txt
- name: Reserve `full`, `regexp` and `keyword` type of rules from custom lists to "reserve" files
run: |
curl -sSL ${CUSTOM_DIRECT} | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > direct-reserve.txt
curl -sSL ${CUSTOM_PROXY} | grep -Ev ":@cn" | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > proxy-reserve.txt
- name: Add proxy, direct and reject domains from "hidden" branch to appropriate temp files
run: |
cat proxy.txt >> temp-proxy.txt
cat direct.txt >> temp-direct.txt
cat reject.txt >> temp-reject.txt
- name: Sort and generate redundant lists
run: |
cat temp-proxy.txt | sort --ignore-case -u > proxy-list-with-redundant
cat temp-direct.txt | sort --ignore-case -u > direct-list-with-redundant
cat temp-reject.txt | sort --ignore-case -u > reject-list-with-redundant
- name: Remove redundant domains
run: |
chmod +x findRedundantDomain.py
./findRedundantDomain.py ./direct-list-with-redundant ./direct-list-deleted-unsort
./findRedundantDomain.py ./proxy-list-with-redundant ./proxy-list-deleted-unsort
./findRedundantDomain.py ./reject-list-with-redundant ./reject-list-deleted-unsort
[ ! -f "direct-list-deleted-unsort" ] && touch direct-list-deleted-unsort
[ ! -f "proxy-list-deleted-unsort" ] && touch proxy-list-deleted-unsort
[ ! -f "reject-list-deleted-unsort" ] && touch reject-list-deleted-unsort
sort ./direct-list-deleted-unsort > ./direct-list-deleted-sort
sort ./proxy-list-deleted-unsort > ./proxy-list-deleted-sort
sort ./reject-list-deleted-unsort > ./reject-list-deleted-sort
diff ./direct-list-deleted-sort ./direct-list-with-redundant | awk '/^>/{print $2}' > ./direct-list-without-redundant
diff ./proxy-list-deleted-sort ./proxy-list-with-redundant | awk '/^>/{print $2}' > ./proxy-list-without-redundant
diff ./reject-list-deleted-sort ./reject-list-with-redundant | awk '/^>/{print $2}' > ./reject-list-without-redundant
- name: Remove domains from "need-to-remove" lists in "hidden" branch
run: |
diff ./direct-need-to-remove.txt ./direct-list-without-redundant | awk '/^>/{print $2}' > temp-cn.txt
diff ./proxy-need-to-remove.txt ./proxy-list-without-redundant | awk '/^>/{print $2}' > temp-geolocation-\!cn.txt
diff ./reject-need-to-remove.txt ./reject-list-without-redundant | awk '/^>/{print $2}' > temp-category-ads-all.txt
- name: Remove domains end with ".cn" in "temp-geolocation-!cn.txt" and write lists to data directory
run: |
cat temp-cn.txt | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/cn
cat temp-cn.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > direct-tld-list.txt
cat temp-geolocation-\!cn.txt | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' | perl -ne 'print if not /\.cn$/' > ./community/data/geolocation-\!cn
cat temp-geolocation-\!cn.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > proxy-tld-list.txt
cat temp-category-ads-all.txt | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/category-ads-all
cat temp-category-ads-all.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > reject-tld-list.txt
- name: Add `full`, `regexp` and `keyword` type of rules back into "cn", "geolocation-!cn" and "category-ads-all" list
run: |
[ -f "direct-reserve.txt" ] && cat direct-reserve.txt >> ./community/data/cn
[ -f "proxy-reserve.txt" ] && cat proxy-reserve.txt >> ./community/data/geolocation-\!cn
[ -f "reject-reserve.txt" ] && cat reject-reserve.txt >> ./community/data/category-ads-all
cp ./community/data/cn direct-list.txt
cp ./community/data/geolocation-\!cn proxy-list.txt
cp ./community/data/category-ads-all reject-list.txt
- name: Create `google-cn`、`apple-cn`、`gfw`、`greatfire` lists
run: |
curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > ./community/data/google-cn
curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > google-cn.txt
curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > ./community/data/apple-cn
curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > apple-cn.txt
cat ./gfwlist2dnsmasq/temp-gfwlist.txt | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> ./community/data/gfw
cat ./community/data/gfw | sort --ignore-case -u > gfw.txt
curl -sSL $GREATFIRE_DOMAINS_URL | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> ./community/data/greatfire
cat ./community/data/greatfire | sort --ignore-case -u > greatfire.txt
curl -sSL $WIN_SPY | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-spy
curl -sSL $WIN_SPY | grep "0.0.0.0" | awk '{print $2}' > win-spy.txt
curl -sSL $WIN_UPDATE | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-update
curl -sSL $WIN_UPDATE | grep "0.0.0.0" | awk '{print $2}' > win-update.txt
curl -sSL $WIN_EXTRA | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-extra
curl -sSL $WIN_EXTRA | grep "0.0.0.0" | awk '{print $2}' > win-extra.txt
curl -sSL $OISD_FULL | grep "0.0.0.0" | awk '{print $2}' > ./community/data/oisd-full
curl -sSL $OISD_FULL | grep "0.0.0.0" | awk '{print $2}' > oisd-full.txt
curl -sSL $OISD_NSFW | grep "0.0.0.0" | awk '{print $2}' > ./community/data/oisd-nsfw
curl -sSL $OISD_NSFW | grep "0.0.0.0" | awk '{print $2}' > oisd-nsfw.txt
wget -O ./community/data/rule-ads $RULE_ADS
wget -O rule-ads.txt $RULE_ADS
wget -O ./community/data/rule-doh $DOH_DNS
wget -O rule-doh.txt $DOH_DNS
wget -O ./community/data/rule-gaming $RULE_GAME
wget -O rule-gaming.txt $RULE_GAME
wget -O ./community/data/rule-indo $RULE_INDO
wget -O rule-indo.txt $RULE_INDO
wget -O ./community/data/rule-playstore $RULE_PS
wget -O rule-playstore.txt $RULE_PS
wget -O ./community/data/rule-sosmed $RULE_SOSMED
wget -O rule-sosmed.txt $RULE_SOSMED
wget -O ./community/data/rule-streaming $RULE_STREAMING
wget -O rule-streaming.txt $RULE_STREAMING
wget -O ./community/data/rule-umum $RULE_UMUM
wget -O rule-umum.txt $RULE_UMUM
wget -O ./community/data/rule-ipcheck $RULE_IPCHECK
wget -O rule-ipcheck.txt $RULE_IPCHECK
wget -O ./community/data/videoconference $RULE_VIDCONFERENCE
wget -O videoconference.txt $RULE_VIDCONFERENCE
wget -O ./community/data/rule-malicious $MALICIOUS_HOST
wget -O rule-malicious.txt $MALICIOUS_HOST
- name: Build geosite.dat file
run: |
cd custom || exit 1
go run ./ --datapath=../community/data
- name: Move and zip files and generate sha256 hash
run: |
install -Dp ./geoip.dat ./publish/geoip.dat
install -Dp ./geoip.dat.sha256sum ./publish/geoip.dat.sha256sum
install -Dp ./custom/publish/geosite.dat ./publish/geosite.dat
cd ./publish || exit 1
zip iptunnels-geosite.zip geoip.dat geosite.dat
sha256sum geosite.dat > geosite.dat.sha256sum
sha256sum iptunnels-geosite.zip > iptunnels-geosite.zip.sha256sum
- name: Release and upload assets
uses: softprops/[email protected]
with:
name: ${{ env.RELEASE_NAME }}
tag_name: ${{ env.TAG_NAME }}
draft: false
prerelease: false
files: |
./publish/*
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Git push assets to "release" branch
run: |
cd publish || exit 1
git init
git config --local user.name "github-actions[bot]"
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -b release
git add .
git commit -m "${{ env.RELEASE_NAME }}"
git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}"
git push -f -u origin release
- name: Purge jsdelivr CDN
run: |
cd publish || exit 1
for file in $(ls); do
curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@release/${file}"
done