|
| 1 | +/******************************************************************************* |
| 2 | +
|
| 3 | + uBlock Origin - a browser extension to block requests. |
| 4 | + Copyright (C) 2022-present Raymond Hill |
| 5 | +
|
| 6 | + This program is free software: you can redistribute it and/or modify |
| 7 | + it under the terms of the GNU General Public License as published by |
| 8 | + the Free Software Foundation, either version 3 of the License, or |
| 9 | + (at your option) any later version. |
| 10 | +
|
| 11 | + This program is distributed in the hope that it will be useful, |
| 12 | + but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | + GNU General Public License for more details. |
| 15 | +
|
| 16 | + You should have received a copy of the GNU General Public License |
| 17 | + along with this program. If not, see {http://www.gnu.org/licenses/}. |
| 18 | +
|
| 19 | + Home: https://github.com/gorhill/uBlock |
| 20 | +*/ |
| 21 | + |
| 22 | +'use strict'; |
| 23 | + |
| 24 | +/******************************************************************************/ |
| 25 | + |
| 26 | +import fs from 'fs/promises'; |
| 27 | +import process from 'process'; |
| 28 | + |
| 29 | +import rulesetConfigs from './ruleset-config.js'; |
| 30 | +import { dnrRulesetFromRawLists } from './js/static-dnr-filtering.js'; |
| 31 | + |
| 32 | +/******************************************************************************/ |
| 33 | + |
| 34 | +const commandLineArgs = (( ) => { |
| 35 | + const args = new Map(); |
| 36 | + let name, value; |
| 37 | + for ( const arg of process.argv.slice(2) ) { |
| 38 | + const pos = arg.indexOf('='); |
| 39 | + if ( pos === -1 ) { |
| 40 | + name = arg; |
| 41 | + value = ''; |
| 42 | + } else { |
| 43 | + name = arg.slice(0, pos); |
| 44 | + value = arg.slice(pos+1); |
| 45 | + } |
| 46 | + args.set(name, value); |
| 47 | + } |
| 48 | + return args; |
| 49 | +})(); |
| 50 | + |
| 51 | +/******************************************************************************/ |
| 52 | + |
| 53 | +async function main() { |
| 54 | + |
| 55 | + const writeOps = []; |
| 56 | + const ruleResources = []; |
| 57 | + const regexRuleResources = []; |
| 58 | + const outputDir = commandLineArgs.get('output') || '.'; |
| 59 | + |
| 60 | + let goodTotalCount = 0; |
| 61 | + let maybeGoodTotalCount = 0; |
| 62 | + |
| 63 | + const output = []; |
| 64 | + const log = (text, silent = false) => { |
| 65 | + output.push(text); |
| 66 | + if ( silent === false ) { |
| 67 | + console.log(text); |
| 68 | + } |
| 69 | + }; |
| 70 | + |
| 71 | + const replacer = (k, v) => { |
| 72 | + if ( k.startsWith('__') ) { return; } |
| 73 | + if ( Array.isArray(v) ) { |
| 74 | + return v.sort(); |
| 75 | + } |
| 76 | + if ( v instanceof Object ) { |
| 77 | + const sorted = {}; |
| 78 | + for ( const kk of Object.keys(v).sort() ) { |
| 79 | + sorted[kk] = v[kk]; |
| 80 | + } |
| 81 | + return sorted; |
| 82 | + } |
| 83 | + return v; |
| 84 | + }; |
| 85 | + |
| 86 | + const isUnsupported = rule => |
| 87 | + rule._error !== undefined; |
| 88 | + const isRegex = rule => |
| 89 | + rule.condition !== undefined && |
| 90 | + rule.condition.regexFilter !== undefined; |
| 91 | + const isRedirect = rule => |
| 92 | + rule.action !== undefined && |
| 93 | + rule.action.type === 'redirect' && |
| 94 | + rule.action.redirect.extensionPath !== undefined; |
| 95 | + const isCsp = rule => |
| 96 | + rule.action !== undefined && |
| 97 | + rule.action.type === 'modifyHeaders'; |
| 98 | + const isRemoveparam = rule => |
| 99 | + rule.action !== undefined && |
| 100 | + rule.action.type === 'redirect' && |
| 101 | + rule.action.redirect.transform !== undefined; |
| 102 | + const isGood = rule => |
| 103 | + isUnsupported(rule) === false && |
| 104 | + isRedirect(rule) === false && |
| 105 | + isCsp(rule) === false && |
| 106 | + isRemoveparam(rule) === false |
| 107 | + ; |
| 108 | + |
| 109 | + const rulesetDir = `${outputDir}/rulesets`; |
| 110 | + const rulesetDirPromise = fs.mkdir(`${rulesetDir}`, { recursive: true }); |
| 111 | + |
| 112 | + const fetchList = url => { |
| 113 | + return fetch(url) |
| 114 | + .then(response => response.text()) |
| 115 | + .then(text => ({ name: url, text })); |
| 116 | + }; |
| 117 | + |
| 118 | + const readList = path => |
| 119 | + fs.readFile(path, { encoding: 'utf8' }) |
| 120 | + .then(text => ({ name: path, text })); |
| 121 | + |
| 122 | + const writeFile = (path, data) => |
| 123 | + rulesetDirPromise.then(( ) => |
| 124 | + fs.writeFile(path, data)); |
| 125 | + |
| 126 | + for ( const ruleset of rulesetConfigs ) { |
| 127 | + const lists = []; |
| 128 | + |
| 129 | + log(`Listset for '${ruleset.id}':`); |
| 130 | + |
| 131 | + if ( Array.isArray(ruleset.paths) ) { |
| 132 | + for ( const path of ruleset.paths ) { |
| 133 | + log(`\t${path}`); |
| 134 | + lists.push(readList(`assets/${path}`)); |
| 135 | + } |
| 136 | + } |
| 137 | + if ( Array.isArray(ruleset.urls) ) { |
| 138 | + for ( const url of ruleset.urls ) { |
| 139 | + log(`\t${url}`); |
| 140 | + lists.push(fetchList(url)); |
| 141 | + } |
| 142 | + } |
| 143 | + |
| 144 | + const rules = await dnrRulesetFromRawLists(lists, { |
| 145 | + env: [ 'chromium' ], |
| 146 | + }); |
| 147 | + |
| 148 | + log(`Ruleset size for '${ruleset.id}': ${rules.length}`); |
| 149 | + |
| 150 | + const good = rules.filter(rule => isGood(rule) && isRegex(rule) === false); |
| 151 | + log(`\tGood: ${good.length}`); |
| 152 | + |
| 153 | + const regexes = rules.filter(rule => isGood(rule) && isRegex(rule)); |
| 154 | + log(`\tMaybe good (regexes): ${regexes.length}`); |
| 155 | + |
| 156 | + const redirects = rules.filter(rule => |
| 157 | + isUnsupported(rule) === false && |
| 158 | + isRedirect(rule) |
| 159 | + ); |
| 160 | + log(`\tredirect-rule= (discarded): ${redirects.length}`); |
| 161 | + |
| 162 | + const headers = rules.filter(rule => |
| 163 | + isUnsupported(rule) === false && |
| 164 | + isCsp(rule) |
| 165 | + ); |
| 166 | + log(`\tcsp= (discarded): ${headers.length}`); |
| 167 | + |
| 168 | + const removeparams = rules.filter(rule => |
| 169 | + isUnsupported(rule) === false && |
| 170 | + isRemoveparam(rule) |
| 171 | + ); |
| 172 | + log(`\tremoveparams= (discarded): ${removeparams.length}`); |
| 173 | + |
| 174 | + const bad = rules.filter(rule => |
| 175 | + isUnsupported(rule) |
| 176 | + ); |
| 177 | + log(`\tUnsupported: ${bad.length}`); |
| 178 | + log( |
| 179 | + bad.map(rule => rule._error.map(v => `\t\t${v}`)).join('\n'), |
| 180 | + true |
| 181 | + ); |
| 182 | + |
| 183 | + writeOps.push( |
| 184 | + writeFile( |
| 185 | + `${rulesetDir}/${ruleset.id}.json`, |
| 186 | + `${JSON.stringify(good, replacer, 2)}\n` |
| 187 | + ) |
| 188 | + ); |
| 189 | + |
| 190 | + regexRuleResources.push({ |
| 191 | + id: ruleset.id, |
| 192 | + enabled: ruleset.enabled, |
| 193 | + rules: regexes |
| 194 | + }); |
| 195 | + |
| 196 | + ruleResources.push({ |
| 197 | + id: ruleset.id, |
| 198 | + enabled: ruleset.enabled, |
| 199 | + path: `/rulesets/${ruleset.id}.json` |
| 200 | + }); |
| 201 | + |
| 202 | + goodTotalCount += good.length; |
| 203 | + maybeGoodTotalCount += regexes.length; |
| 204 | + } |
| 205 | + |
| 206 | + writeOps.push( |
| 207 | + writeFile( |
| 208 | + `${rulesetDir}/regexes.js`, |
| 209 | + `export default ${JSON.stringify(regexRuleResources, replacer, 2)};\n` |
| 210 | + ) |
| 211 | + ); |
| 212 | + |
| 213 | + await Promise.all(writeOps); |
| 214 | + |
| 215 | + log(`Total good rules count: ${goodTotalCount}`); |
| 216 | + log(`Total regex rules count: ${maybeGoodTotalCount}`); |
| 217 | + |
| 218 | + // Patch manifest |
| 219 | + const manifest = await fs.readFile(`${outputDir}/manifest.json`, { encoding: 'utf8' }) |
| 220 | + .then(text => JSON.parse(text)); |
| 221 | + manifest.declarative_net_request = { rule_resources: ruleResources }; |
| 222 | + const now = new Date(); |
| 223 | + manifest.version = `0.1.${now.getUTCFullYear() - 2000}.${now.getUTCMonth() * 100 + now.getUTCDate()}`; |
| 224 | + await fs.writeFile( |
| 225 | + `${outputDir}/manifest.json`, |
| 226 | + JSON.stringify(manifest, null, 2) + '\n' |
| 227 | + ); |
| 228 | + |
| 229 | + // Log results |
| 230 | + await fs.writeFile(`${outputDir}/log.txt`, output.join('\n') + '\n'); |
| 231 | +} |
| 232 | + |
| 233 | +main(); |
| 234 | + |
| 235 | +/******************************************************************************/ |
0 commit comments