From 695b48b1d062876a6a1ed643f85e17ac41a44cf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Deckert?= Date: Fri, 21 Mar 2025 19:08:46 +0100 Subject: [PATCH] add: net-proxy/squid-opnsense-fetchacls, --- app-emulation/open-vmdk/Manifest | 2 +- .../open-vmdk/open-vmdk-20211104.ebuild | 4 +- app-portage/virtual-appliance/Manifest | 4 +- ...=> virtual-appliance-2.0-r20250321.ebuild} | 3 +- net-proxy/squid-opnsense-fetchacls/Manifest | 3 +- .../files/fetchACLs-github-download.patch | 145 ++++++++++++++++++ .../squid-opnsense-fetchacls-24.7.12.ebuild | 7 +- 7 files changed, 160 insertions(+), 8 deletions(-) rename app-portage/virtual-appliance/{virtual-appliance-2.0-r20230715.ebuild => virtual-appliance-2.0-r20250321.ebuild} (96%) create mode 100644 net-proxy/squid-opnsense-fetchacls/files/fetchACLs-github-download.patch diff --git a/app-emulation/open-vmdk/Manifest b/app-emulation/open-vmdk/Manifest index 2b3f4f6..048c03d 100644 --- a/app-emulation/open-vmdk/Manifest +++ b/app-emulation/open-vmdk/Manifest @@ -1,3 +1,3 @@ AUX diskinfo-20211104.patch 701 BLAKE2B 5f9d03a2154f0b21d1e83eeeb340c4512394558668f7d119ce8616c5021ff2f5fc3da5b21e0e4bd9c4defd66f6d00e6647fe81973fdb6f8c2b99682861a0fb74 SHA512 f5895dd293bfdb58dd87d9540fd05cc39018620284fef8c08b2bcc09da3a4aa319ed0451ee1d8f8a3c405d7cfb262b59a5845873cf216093583593aafa2fe512 DIST open-vmdk-20211104.zip 47295 BLAKE2B 52e656c2ffd4ca88c2cb598225ce0e6f08415e683078ea77ef5cd7045ae766582a47acbb70e7dab8649f812f768b5671d32e1b3bc0f48ce2673607a1035a97ee SHA512 604e722146ca39cd932824c0a2d35c467bc1b931d9ff16022c48d2840b48c3731b49629584497d0d98437d406febb6a2ccc5ee4278c08708782fc13280031ca4 -EBUILD open-vmdk-20211104.ebuild 836 BLAKE2B 890a4013ce7447ccc6794a96a255aba76e7ad2991a90da547ad24c9708c96bfb921d3775b03e7f02c624b28c19503f054d5d4532cd6cb7965e27b2bc73f08224 SHA512 de088cf4b4e4c04b4596c5c2d5d191c76930536c08ab49137637a12ac1e862154be1af322eac6be4233f16bad0d31e7ae9f9413f12c87e01a1d64efcaa447c4c +EBUILD open-vmdk-20211104.ebuild 842 BLAKE2B 02b8be909e3c2a486b75bb8a447eaed34b949d666a593fe982afc162aaca9b91cce2584d5244e1388659f560f6ac754ea08f1129da03f0c47fc7d496eeb08e2b SHA512 f94b10c3497822f1219b328e8597cf461638394699006e9a257f4d2a34c01bce7d9c348b39fbb994ec3dcb4f3179805451ac057dc403174334a3b5aaf9d2f08f diff --git a/app-emulation/open-vmdk/open-vmdk-20211104.ebuild b/app-emulation/open-vmdk/open-vmdk-20211104.ebuild index 2951c68..2a7ee70 100644 --- a/app-emulation/open-vmdk/open-vmdk-20211104.ebuild +++ b/app-emulation/open-vmdk/open-vmdk-20211104.ebuild @@ -1,10 +1,10 @@ # $Header: $ -EAPI="7" +EAPI="8" COMMIT="875f4162c91c7fc7bc450dccaf7b896a927fa42b" -PYTHON_COMPAT=( python3_{7,8,9,10,11} ) +PYTHON_COMPAT=( python3_{7,8,9,10,11,12,13} ) inherit python-r1 DESCRIPTION="Convert VMware .vmdk images, build .ovf and .ova" diff --git a/app-portage/virtual-appliance/Manifest b/app-portage/virtual-appliance/Manifest index e1a496a..16b98ca 100644 --- a/app-portage/virtual-appliance/Manifest +++ b/app-portage/virtual-appliance/Manifest @@ -1,2 +1,2 @@ -DIST virtual-appliance-2.0-r20230715.tar.gz 55573 BLAKE2B bb51e9cc98fd46ce6a0be278b2fd05f20d842f8f6131b77e7a01b3398d225b40a1adc836bc5abad69faa5c98efc4bd114af3633cde00b1416cc93f42ff1f59ed SHA512 3d1317662faff3d431f7589f4cd16efcffadd5b3caf42f79162ce81332b33ac6adacff255fd41bf29e195d0bc3f47502ea3d36c39b6c87b4bf23576174098c4f -EBUILD virtual-appliance-2.0-r20230715.ebuild 1102 BLAKE2B ac48609fb8060a132125fa502a46bb871179aaac1e70ad7a7eec41f1f40925f90830f536fd190e053b0f33a62140630974a321e8f6135bc77cae2e04d7412511 SHA512 140855469fafa51c4aa520875275331722d8eacf5b9ba3a5dfb518fcd98bc3b974cc05bb737d4c6d1b2c626a5529cbaf6f17093ad43a37a396c6b83441a3baad +DIST virtual-appliance-2.0-r20250321.tar.gz 56870 BLAKE2B 1a49ae4d62892624c187c30b32d1873927a3aa65e413afeba919ca13fd22c76096ac5abe3f442d4835d5df9ed28a3792d7db8501239f3d1e8bc9263f71628b62 SHA512 f882fad772cf4235cf2129ae290323279dd28b112444a1d1a0a9f18cbd13112a22ef47e9bbd70b3041b99751011cd6fd3f8d930db20e7b0c18be88a8c7ca2194 +EBUILD virtual-appliance-2.0-r20250321.ebuild 1134 BLAKE2B 48772be9cd1a6fd063c7339541c05d35a2ccf42c875612bcf7b96c3954310c8d85961b8d95166a00b5750c66a9b527265db518ca192540422ab240d9573ef7ae SHA512 6b42452293cef9daf1c35ee4225e928615abc4a249413141e699b720f88f453e74656bda102b3ebb8f7e4945ab9e1f9535c6c43929120e6a7d02a6093eb5fd32 diff --git a/app-portage/virtual-appliance/virtual-appliance-2.0-r20230715.ebuild b/app-portage/virtual-appliance/virtual-appliance-2.0-r20250321.ebuild similarity index 96% rename from app-portage/virtual-appliance/virtual-appliance-2.0-r20230715.ebuild rename to app-portage/virtual-appliance/virtual-appliance-2.0-r20250321.ebuild index 7f855e5..9416b7c 100644 --- a/app-portage/virtual-appliance/virtual-appliance-2.0-r20230715.ebuild +++ b/app-portage/virtual-appliance/virtual-appliance-2.0-r20250321.ebuild @@ -2,7 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 # $Header: $ -EAPI=7 +EAPI=8 if [[ ${PV} == 9999* ]]; then EGIT_REPO_URI="https://dev.unitas-network.de/r/VA/${PN}.git" @@ -25,6 +25,7 @@ IUSE="" DEPEND="app-emulation/qemu app-emulation/open-vmdk + dev-python/python-gnupg sys-block/parted" RDEPEND="${DEPEND}" diff --git a/net-proxy/squid-opnsense-fetchacls/Manifest b/net-proxy/squid-opnsense-fetchacls/Manifest index 98f0e9c..2435d7c 100644 --- a/net-proxy/squid-opnsense-fetchacls/Manifest +++ b/net-proxy/squid-opnsense-fetchacls/Manifest @@ -1,8 +1,9 @@ AUX externalACLs.conf 167 BLAKE2B 53e5447dc37ab8b37b24e5089dbb39193ccb68334ac24835066c26279120ff9c56db135024c44e5460cd35f1e81db7c0862415b8c36e61398e2ce379fe278480 SHA512 8ee725709e497ff4f6651718e5d955268a50f98b992f04db2021f48029d41946503e7d1adc4e7e4cb543405105490523f6ab3bcffd9c700ffe3a96025a7af7dd +AUX fetchACLs-github-download.patch 7597 BLAKE2B 754567cdbc1fe4881a3240878cdd49c012c25b5ea23c3dd192ad99b66edf7b1763cd3eb7c397ab5793c887f349ce284be59ece06c78589c3a6546d2220212ce5 SHA512 28fc58a322d244d9a3b90ba5a0207a81e3816cff382209de507bce766621341f7163b48108ff90f34e0773885032d95d592d230430e01bebc688879ea65bb523 AUX fetchACLs.service 153 BLAKE2B cff1dff12195aa0ae2b9681278c5a71f851654395cfef46ca5830a719a0417ce0dda52479a08b6a151c6aa51bbfc0edd580cbcdc649ca0e6ad167b4585747df4 SHA512 95e5762e65af8c057e1927808e79f22edd7e1608509fd50201468a38cbd0efd676e30d2089321d7615f5a590aae0cd3c15f4fb1165122061c4d538176e3072dd AUX fetchACLs.timer 210 BLAKE2B a1906ec839d41d858d64b17ef05c52581d3bf8e4d1c7b228b3c902bd52b669da6c3d28afdfbcd073cc379b83dcb9b279e787ddc99e070c2743804bb9b37dadf6 SHA512 7ea76d32892c85dffcb156f2ba04a83b9f2c7683729ba9884badbeec9d7b977f9eb759cbf6835768b23826ae09602d24b2d43443ad461c85c8f5d5b804e284c0 AUX squid.conf.NoBumping 3864 BLAKE2B f4579d23289288b29381597fae87eacd0879bb8ef9ac2e856ff5a2e0128da15d4839195b6533a1f36fffb8da1c528fddcc6ce5896cf1597fbb1edeb6917fd921 SHA512 7bb25ab02652ce9ba19ba99d2d7b2c39e54e92a044dd24fb7f1d1bdce806005d57fc5e6fe5efa8669bf7f0ce1007514c88af5ee51bc5ce4282cddaf4ca037e8c AUX squid.conf.SSLBump 4261 BLAKE2B 4e75f2d997f49d2e67ad1b6d0c9de12087bcba01b1a7504f63ed8862b3774855a060a01ef23e87bb0581c092e09e3d4f6a58cbd14b4b60092d94dad19676a341 SHA512 6d0aacc392dc581addd35052d16daa215ededb2421b4873695aad1f291fa44b8397b899dd478e5e374b1c20fdd3998ee313c65c1797eaa60a4d9b4ab5f20415c AUX squid.conf.Transparent 3645 BLAKE2B 3485c37e8cd4ff4b3cc55a8d188877cafea5ed178f7a5e7cb488d98849ec67f9b58b49100a13eae2294136dddc219f93e32d3a69209abeeb0292149d2df46c80 SHA512 e60b502b6e1e49ae77d9ef9d1c5b979cc7c708d6517261be984acbf9c4a63465346b813b3cb87224f9764c1d478b5433e900dbbdf1323241710e2de5b28f0865 DIST squid-opnsense-fetchacls-24.7.12.py 16231 BLAKE2B 391fa8a5808b2fd3100a8ba52d1a70105819329ee6bc7ee31dcc9717934d7ac0fea64bed73b0288931fc26697dd3b5c95275ea83e21863c898a090a824129d15 SHA512 104ee310add5f61e58afe5324db7677d113e25e6d20b6d1a5c0f185c1b358ce6a805346a6f8080c028ae2671a83a4e35a0f9f2dcd00bfb4c3b9ea0813489544e -EBUILD squid-opnsense-fetchacls-24.7.12.ebuild 1131 BLAKE2B 6816347d7936e51dece3a019efd785d6fd2c07dd681216e7f5899ef0d898b9f8677895e54ae1df095a1b8548392b7e11cb5159f6545b1ea4c8174b1b29531581 SHA512 30873af4e5f345ab606722e6e8959f2862770f8a59e84aac3055a4f0ea4e729421f535754d1c4285e6b8eaf30aa2c7c6638b7ec4349c372e6b16a279ffb6d1ce +EBUILD squid-opnsense-fetchacls-24.7.12.ebuild 1306 BLAKE2B 6e547d073365cfec6e589f9d0e6227fd75260a96bb07664af472bfbed906662f608b6bf252a42c89de2623a019b83cec5047b7b976ad7d78eb4f3c17866c495a SHA512 8ca7edcd63173f2ba25c786ba1a0ed8f96c93617317cea48a0c76c05614eca4590fb09916cc3e01b0cff4bb924837176b68484a9d50d5cee30622e6b838f5144 diff --git a/net-proxy/squid-opnsense-fetchacls/files/fetchACLs-github-download.patch b/net-proxy/squid-opnsense-fetchacls/files/fetchACLs-github-download.patch new file mode 100644 index 0000000..5ec33ce --- /dev/null +++ b/net-proxy/squid-opnsense-fetchacls/files/fetchACLs-github-download.patch @@ -0,0 +1,145 @@ +--- squid-opnsense-fetchacls-24.7.12.py 2025-02-03 09:16:20.922603218 +0100 ++++ fetchACLs.py 2025-02-03 09:38:51.971220869 +0100 +@@ -54,7 +54,7 @@ + """ Download helper + """ + +- def __init__(self, url,username, password, timeout, ssl_no_verify=False): ++ def __init__(self, url,username, password, timeout, acl_list, ssl_no_verify=False): + """ init new + :param url: source url + :param timeout: timeout in seconds +@@ -65,6 +65,7 @@ + self._username = username + self._password = password + self._ssl_no_verify = ssl_no_verify ++ self._acl_list = acl_list + + def fetch(self): + """ fetch (raw) source data into tempfile using self._source_handle +@@ -115,37 +116,24 @@ + def get_files(self): + """ process downloaded data, handle compression + :return: iterator filename, file handle ++ zip-function is written for github-blocklists (ZIP-file including mutiple directories with actual blocklists ++ e.g. ut1-blocklists-master/blocklists/adult/domains + """ + if self._source_handle is not None: +- # handle compressed data +- if (len(self._url) > 8 and self._url[-7:] == '.tar.gz') \ +- or (len(self._url) > 4 and self._url[-4:] == '.tgz'): +- # source is in tar.gz format, extract all into a single string +- try: +- tf = tarfile.open(fileobj=self._source_handle) +- for tf_file in tf.getmembers(): +- if tf_file.isfile(): +- yield tf_file.name, tf.extractfile(tf_file) +- except IOError as e: +- syslog.syslog(syslog.LOG_ERR, 'proxy acl: error downloading %s (%s)' % (self._url, e)) +- elif len(self._url) > 4 and self._url[-3:] == '.gz': +- # source is in .gz format unpack +- try: +- gf = gzip.GzipFile(mode='r', fileobj=self._source_handle) +- yield os.path.basename(self._url), gf +- except IOError as e: +- syslog.syslog(syslog.LOG_ERR, 'proxy acl: error downloading %s (%s)' % (self._url, e)) +- elif len(self._url) > 5 and self._url[-4:] == '.zip': +- # source is in .zip format, extract all into a single string +- with zipfile.ZipFile(self._source_handle, +- mode='r', +- compression=zipfile.ZIP_DEFLATED) as zf: +- for item in zf.infolist(): +- if item.file_size > 0: +- yield item.filename, zf.open(item) +- else: +- yield os.path.basename(self._url), self._source_handle +- ++ if len(self._url) > 5 and self._url[-4:] == '.zip': ++ with zipfile.ZipFile(self._source_handle, mode='r') as zip: ++ for item in zip.infolist(): ++ if item.filename.startswith('ut1-blacklists-master/blacklists/'): ++ if check_filter(self, item.filename): ++ if "domains" in item.filename: #zip contains absolute paths: only if path starts with ...blocklists and ends with domain ++ with zip.open(item.filename) as file: #every domain-file is opened ++ print(item.filename) ++ content = file.read() ++ content = content.decode('utf-8', errors='ignore') #file-content is parsed into variable and gets decoded to utf-8 ++ yield item.filename, content ++ elif (len(self._url) > 8 and self._url[-7:] == '.tar.gz') or (len(self._url) > 4 and self._url[-4:] == '.tgz'): ++ tar = tarfile.open(fileobj=self._source_handle) ++ yield from extract_tar(self, tar) + def download(self): + """ download / unpack ACL + :return: iterator filename, type, content +@@ -154,12 +142,40 @@ + for filename, filehandle in self.get_files(): + basefilename = os.path.basename(filename).lower() + file_ext = filename.split('.')[-1].lower() +- while True: +- line = filehandle.readline().decode(encoding='utf-8', errors='ignore') +- if not line: +- break +- yield filename, basefilename, file_ext, line ++ for line in filehandle.splitlines(): ++ line = line.strip() ++ if line: ++ yield filename, basefilename, file_ext, line ++ ++def check_filter(obj, filename): ++ acl_list = obj._acl_list ++ domain = filename.split('/')[-2].lower() #start from end of array and get second last element ++ if len(acl_list) > 0: ++ if domain in acl_list: ++ return True ++ else: ++ return False ++ else: ++ return True + ++def extract_tar(obj, tar_file, parent_dir=''): ++ for tf_file in tar_file.getmembers(): ++ file_name = tf_file.name ++ if tf_file.isfile() and (file_name.endswith('.tar.gz') or file_name.endswith('.tgz')): ++ try: ++ inner_file = tar_file.extractfile(tf_file) ++ inner_tar = tarfile.open(fileobj=inner_file) ++ yield from extract_tar(obj, inner_tar, parent_dir + tf_file.name + '/') ++ except Exception as e: ++ syslog.syslog(syslog.LOG_ERR, 'proxy acl: error downloading or extracting tarball: %s (%s)' % (obj._url, e)) ++ elif tf_file.isfile() and not tf_file.name.endswith("."): ++ if "domains" in tf_file.name: ++ if check_filter(obj, tf_file.name): ++ print(tf_file.name) ++ content = tar_file.extractfile(tf_file).read().decode('utf-8', errors='ignore') ++ yield tf_file.name, content ++ else: ++ continue + + class DomainSorter(object): + """ Helper class for building sorted squid domain acl list. +@@ -320,7 +336,7 @@ + sslNoVerify = True + else: + sslNoVerify = False +- acl = Downloader(download_url, download_username, download_password, acl_max_timeout, sslNoVerify) ++ acl = Downloader(download_url, download_username, download_password, acl_max_timeout, acl_filters, sslNoVerify) + all_filenames = list() + for filename, basefilename, file_ext, line in acl.download(): + if filename_in_ignorelist(basefilename, file_ext): +@@ -338,16 +354,6 @@ + if filename not in all_filenames: + all_filenames.append(filename) + +- if len(acl_filters) > 0: +- acl_found = False +- for acl_filter in acl_filters: +- if acl_filter in filename: +- acl_found = True +- break +- if not acl_found: +- # skip this acl entry +- continue +- + if filetype in targets and targets[filetype]['handle'] is None: + targets[filetype]['handle'] = targets[filetype]['class'](targets[filetype]['filename']) + if filetype in targets: diff --git a/net-proxy/squid-opnsense-fetchacls/squid-opnsense-fetchacls-24.7.12.ebuild b/net-proxy/squid-opnsense-fetchacls/squid-opnsense-fetchacls-24.7.12.ebuild index 10f30bc..adb24dd 100644 --- a/net-proxy/squid-opnsense-fetchacls/squid-opnsense-fetchacls-24.7.12.ebuild +++ b/net-proxy/squid-opnsense-fetchacls/squid-opnsense-fetchacls-24.7.12.ebuild @@ -33,13 +33,18 @@ src_prepare() { -e 's|/usr/local/etc/squid/acl|/var/lib/squid/acl|' \ ${P}.py || die + cp ${P}.py fetchACLs.py + eapply -p0 "${FILESDIR}/fetchACLs-github-download.patch" + eapply_user } src_install() { - newbin ${P}.py fetchACLs.py + dobin fetchACLs.py python_replicate_script "${D}"/usr/bin/fetchACLs.py + newbin ${P}.py fetchACLs-OPNsense.py + python_replicate_script "${D}"/usr/bin/fetchACLs-OPNsense.py systemd_dounit "${FILESDIR}"/fetchACLs.{service,timer} insinto /etc/squid doins "${FILESDIR}"/externalACLs.conf "${FILESDIR}"/squid.conf.*