forked from flinteger/dnss-blocklists
-
Notifications
You must be signed in to change notification settings - Fork 1
168 lines (135 loc) · 5.23 KB
/
CD.yml
File metadata and controls
168 lines (135 loc) · 5.23 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
name: CD
on:
push:
branches: [ master ]
schedule:
# build once every 12 hours
- cron: "0 */12 * * *"
# allow manually trigger a build
workflow_dispatch:
jobs:
CD:
# https://github.com/actions/runner-images
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
# Number of commits to fetch. 0 indicates all history for all branches and tags.
# Default: 1
# Only keep the latest 10 history to limit repo size
fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
- name: Prepare cache
run: |
# Create empty cache files to allow `hashFiles` success.
mkdir -p cache
touch cache/nxdomains cache/okdomains cache/tbddomains cache/hash.txt cache/hash_old.txt
# https://github.com/actions/cache
- name: Cache domains status
uses: actions/cache@v3
with:
path: |
cache/nxdomains
cache/okdomains
cache/tbddomains
key: cache-domains-status-${{ hashFiles('cache/hash.txt') }}
restore-keys: cache-domains-status-${{ hashFiles('cache/hash_old.txt') }}
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: 3.11
architecture: x64
- name: Check build env
run: |
echo "pwd: $(pwd)"
echo "USER: $USER"
echo "HOME: $HOME"
echo "SHELL: $SHELL"
echo "PATH: $PATH"
python3 --version
pip3 --version
mkdir -p cache
ls -lh cache
ulimit -a
- name: Setup
run: |
# pip install binary to $HOME/.local/bin
# export PATH="$PATH:$HOME/.local/bin"
#
# https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#adding-a-system-path
echo "/home/runner/.local/bin" >> $GITHUB_PATH
make setup
mkdir -p blocklists
- name: Build blocklists from upstream
run: |
# make all
# ./scripts/process_all.sh
set -x
# Remove wildcard domains from `cache/nxdomains`
if [ -s cache/nxdomains ]; then
# if cache/nxdomains is empty, grep return 1 will cause script failure.
grep -v '*' cache/nxdomains > foo
mv foo cache/nxdomains
fi
./scripts/process.py ./sources/ad.json
./scripts/process.py ./sources/dating.json
./scripts/process.py ./sources/game.json
./scripts/process.py ./sources/gambling.json
./scripts/process.py ./sources/malicious.json
./scripts/process.py ./sources/piracy.json
./scripts/process.py ./sources/porn.json
./scripts/process.py ./sources/social_networks.json
./scripts/process.py ./sources/service.*.json
wget https://openphish.com/feed.txt -O blocklists/phishing.txt
# - run: ./scripts/process.py ./sources/ad.json
# - run: ./scripts/process.py ./sources/dating.json
# - run: ./scripts/process.py ./sources/gambling.json
# - run: ./scripts/process.py ./sources/malicious.json
# - run: ./scripts/process.py ./sources/piracy.json
# - run: ./scripts/process.py ./sources/porn.json
# - run: ./scripts/process.py ./sources/social_networks.json
- name: Check changes and clean cache
run: |
ls -lh blocklists
wc -l blocklists/*.domains.txt | sort -n
./scripts/clean_okcache.py
ls -lh cache/nxdomains cache/okdomains cache/tbddomains cache/alldomains
./scripts/clean_cache.py
ls -lh cache/nxdomains cache/okdomains cache/tbddomains cache/alldomains
mv cache/hash.txt cache/hash_old.txt
sha1sum cache/nxdomains cache/okdomains cache/tbddomains > cache/hash.txt
git status
- name: Git config
run: |
git config --local user.email "CD@github-action"
git config --local user.name "GitHub Action"
- name: Commit changes on release branch
run: |
git checkout --orphan release
./scripts/gen_readme.py > README.md
wget --quiet https://github.com/jgm/pandoc/releases/download/3.0.1/pandoc-3.0.1-linux-amd64.tar.gz
tar zxvf pandoc-3.0.1-linux-amd64.tar.gz
chmod +x ./pandoc-3.0.1/bin/pandoc
./pandoc-3.0.1/bin/pandoc README.md -f markdown -t html -s --metadata title="dnss-blocklist" -o index.html
git add sources blocklists icons README.md index.html 404.html
git commit -m "Update blocklists from upstream" sources blocklists icons README.md index.html 404.html
- name: Force push data branch changes
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
# branch: ${{ github.ref }}
branch: release
force: true
- name: Commit changes on master branch
run: |
git checkout master
git add cache/hash.txt cache/hash_old.txt
git commit -m "Update cache hash" cache/hash.txt cache/hash_old.txt
git fetch
git rebase
- name: Push master branch changes
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: master
force: false