add: net-proxy/squid-opnsense-fetchacls,

2025-03-21 19:08:46 +01:00 · 2025-03-21 19:08:46 +01:00 · 695b48b1d0
commit 695b48b1d0
parent 3e16d6dd3b
7 changed files with 160 additions and 8 deletions
--- a/app-emulation/open-vmdk/Manifest
+++ b/app-emulation/open-vmdk/Manifest
@ -1,3 +1,3 @@
 AUX diskinfo-20211104.patch 701 BLAKE2B 5f9d03a2154f0b21d1e83eeeb340c4512394558668f7d119ce8616c5021ff2f5fc3da5b21e0e4bd9c4defd66f6d00e6647fe81973fdb6f8c2b99682861a0fb74 SHA512 f5895dd293bfdb58dd87d9540fd05cc39018620284fef8c08b2bcc09da3a4aa319ed0451ee1d8f8a3c405d7cfb262b59a5845873cf216093583593aafa2fe512
 DIST open-vmdk-20211104.zip 47295 BLAKE2B 52e656c2ffd4ca88c2cb598225ce0e6f08415e683078ea77ef5cd7045ae766582a47acbb70e7dab8649f812f768b5671d32e1b3bc0f48ce2673607a1035a97ee SHA512 604e722146ca39cd932824c0a2d35c467bc1b931d9ff16022c48d2840b48c3731b49629584497d0d98437d406febb6a2ccc5ee4278c08708782fc13280031ca4
-EBUILD open-vmdk-20211104.ebuild 836 BLAKE2B 890a4013ce7447ccc6794a96a255aba76e7ad2991a90da547ad24c9708c96bfb921d3775b03e7f02c624b28c19503f054d5d4532cd6cb7965e27b2bc73f08224 SHA512 de088cf4b4e4c04b4596c5c2d5d191c76930536c08ab49137637a12ac1e862154be1af322eac6be4233f16bad0d31e7ae9f9413f12c87e01a1d64efcaa447c4c
+EBUILD open-vmdk-20211104.ebuild 842 BLAKE2B 02b8be909e3c2a486b75bb8a447eaed34b949d666a593fe982afc162aaca9b91cce2584d5244e1388659f560f6ac754ea08f1129da03f0c47fc7d496eeb08e2b SHA512 f94b10c3497822f1219b328e8597cf461638394699006e9a257f4d2a34c01bce7d9c348b39fbb994ec3dcb4f3179805451ac057dc403174334a3b5aaf9d2f08f
--- a/app-emulation/open-vmdk/open-vmdk-20211104.ebuild
+++ b/app-emulation/open-vmdk/open-vmdk-20211104.ebuild
@ -1,10 +1,10 @@
 # $Header: $

-EAPI="7"
+EAPI="8"

 COMMIT="875f4162c91c7fc7bc450dccaf7b896a927fa42b"

-PYTHON_COMPAT=( python3_{7,8,9,10,11} )
+PYTHON_COMPAT=( python3_{7,8,9,10,11,12,13} )
 inherit python-r1

 DESCRIPTION="Convert VMware .vmdk images, build .ovf and .ova"
--- a/app-portage/virtual-appliance/Manifest
+++ b/app-portage/virtual-appliance/Manifest
@ -1,2 +1,2 @@
-DIST virtual-appliance-2.0-r20230715.tar.gz 55573 BLAKE2B bb51e9cc98fd46ce6a0be278b2fd05f20d842f8f6131b77e7a01b3398d225b40a1adc836bc5abad69faa5c98efc4bd114af3633cde00b1416cc93f42ff1f59ed SHA512 3d1317662faff3d431f7589f4cd16efcffadd5b3caf42f79162ce81332b33ac6adacff255fd41bf29e195d0bc3f47502ea3d36c39b6c87b4bf23576174098c4f
-EBUILD virtual-appliance-2.0-r20230715.ebuild 1102 BLAKE2B ac48609fb8060a132125fa502a46bb871179aaac1e70ad7a7eec41f1f40925f90830f536fd190e053b0f33a62140630974a321e8f6135bc77cae2e04d7412511 SHA512 140855469fafa51c4aa520875275331722d8eacf5b9ba3a5dfb518fcd98bc3b974cc05bb737d4c6d1b2c626a5529cbaf6f17093ad43a37a396c6b83441a3baad
+DIST virtual-appliance-2.0-r20250321.tar.gz 56870 BLAKE2B 1a49ae4d62892624c187c30b32d1873927a3aa65e413afeba919ca13fd22c76096ac5abe3f442d4835d5df9ed28a3792d7db8501239f3d1e8bc9263f71628b62 SHA512 f882fad772cf4235cf2129ae290323279dd28b112444a1d1a0a9f18cbd13112a22ef47e9bbd70b3041b99751011cd6fd3f8d930db20e7b0c18be88a8c7ca2194
+EBUILD virtual-appliance-2.0-r20250321.ebuild 1134 BLAKE2B 48772be9cd1a6fd063c7339541c05d35a2ccf42c875612bcf7b96c3954310c8d85961b8d95166a00b5750c66a9b527265db518ca192540422ab240d9573ef7ae SHA512 6b42452293cef9daf1c35ee4225e928615abc4a249413141e699b720f88f453e74656bda102b3ebb8f7e4945ab9e1f9535c6c43929120e6a7d02a6093eb5fd32
--- a/app-portage/virtual-appliance/virtual-appliance-2.0-r20250321.ebuild
+++ b/app-portage/virtual-appliance/virtual-appliance-2.0-r20250321.ebuild
@ -2,7 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Header: $

-EAPI=7
+EAPI=8

 if [[ ${PV} == 9999* ]]; then
 	EGIT_REPO_URI="https://dev.unitas-network.de/r/VA/${PN}.git"
@ -25,6 +25,7 @@ IUSE=""

 DEPEND="app-emulation/qemu
        app-emulation/open-vmdk
+        dev-python/python-gnupg
        sys-block/parted"
 RDEPEND="${DEPEND}"

--- a/net-proxy/squid-opnsense-fetchacls/Manifest
+++ b/net-proxy/squid-opnsense-fetchacls/Manifest
@ -1,8 +1,9 @@
 AUX externalACLs.conf 167 BLAKE2B 53e5447dc37ab8b37b24e5089dbb39193ccb68334ac24835066c26279120ff9c56db135024c44e5460cd35f1e81db7c0862415b8c36e61398e2ce379fe278480 SHA512 8ee725709e497ff4f6651718e5d955268a50f98b992f04db2021f48029d41946503e7d1adc4e7e4cb543405105490523f6ab3bcffd9c700ffe3a96025a7af7dd
+AUX fetchACLs-github-download.patch 7597 BLAKE2B 754567cdbc1fe4881a3240878cdd49c012c25b5ea23c3dd192ad99b66edf7b1763cd3eb7c397ab5793c887f349ce284be59ece06c78589c3a6546d2220212ce5 SHA512 28fc58a322d244d9a3b90ba5a0207a81e3816cff382209de507bce766621341f7163b48108ff90f34e0773885032d95d592d230430e01bebc688879ea65bb523
 AUX fetchACLs.service 153 BLAKE2B cff1dff12195aa0ae2b9681278c5a71f851654395cfef46ca5830a719a0417ce0dda52479a08b6a151c6aa51bbfc0edd580cbcdc649ca0e6ad167b4585747df4 SHA512 95e5762e65af8c057e1927808e79f22edd7e1608509fd50201468a38cbd0efd676e30d2089321d7615f5a590aae0cd3c15f4fb1165122061c4d538176e3072dd
 AUX fetchACLs.timer 210 BLAKE2B a1906ec839d41d858d64b17ef05c52581d3bf8e4d1c7b228b3c902bd52b669da6c3d28afdfbcd073cc379b83dcb9b279e787ddc99e070c2743804bb9b37dadf6 SHA512 7ea76d32892c85dffcb156f2ba04a83b9f2c7683729ba9884badbeec9d7b977f9eb759cbf6835768b23826ae09602d24b2d43443ad461c85c8f5d5b804e284c0
 AUX squid.conf.NoBumping 3864 BLAKE2B f4579d23289288b29381597fae87eacd0879bb8ef9ac2e856ff5a2e0128da15d4839195b6533a1f36fffb8da1c528fddcc6ce5896cf1597fbb1edeb6917fd921 SHA512 7bb25ab02652ce9ba19ba99d2d7b2c39e54e92a044dd24fb7f1d1bdce806005d57fc5e6fe5efa8669bf7f0ce1007514c88af5ee51bc5ce4282cddaf4ca037e8c
 AUX squid.conf.SSLBump 4261 BLAKE2B 4e75f2d997f49d2e67ad1b6d0c9de12087bcba01b1a7504f63ed8862b3774855a060a01ef23e87bb0581c092e09e3d4f6a58cbd14b4b60092d94dad19676a341 SHA512 6d0aacc392dc581addd35052d16daa215ededb2421b4873695aad1f291fa44b8397b899dd478e5e374b1c20fdd3998ee313c65c1797eaa60a4d9b4ab5f20415c
 AUX squid.conf.Transparent 3645 BLAKE2B 3485c37e8cd4ff4b3cc55a8d188877cafea5ed178f7a5e7cb488d98849ec67f9b58b49100a13eae2294136dddc219f93e32d3a69209abeeb0292149d2df46c80 SHA512 e60b502b6e1e49ae77d9ef9d1c5b979cc7c708d6517261be984acbf9c4a63465346b813b3cb87224f9764c1d478b5433e900dbbdf1323241710e2de5b28f0865
 DIST squid-opnsense-fetchacls-24.7.12.py 16231 BLAKE2B 391fa8a5808b2fd3100a8ba52d1a70105819329ee6bc7ee31dcc9717934d7ac0fea64bed73b0288931fc26697dd3b5c95275ea83e21863c898a090a824129d15 SHA512 104ee310add5f61e58afe5324db7677d113e25e6d20b6d1a5c0f185c1b358ce6a805346a6f8080c028ae2671a83a4e35a0f9f2dcd00bfb4c3b9ea0813489544e
-EBUILD squid-opnsense-fetchacls-24.7.12.ebuild 1131 BLAKE2B 6816347d7936e51dece3a019efd785d6fd2c07dd681216e7f5899ef0d898b9f8677895e54ae1df095a1b8548392b7e11cb5159f6545b1ea4c8174b1b29531581 SHA512 30873af4e5f345ab606722e6e8959f2862770f8a59e84aac3055a4f0ea4e729421f535754d1c4285e6b8eaf30aa2c7c6638b7ec4349c372e6b16a279ffb6d1ce
+EBUILD squid-opnsense-fetchacls-24.7.12.ebuild 1306 BLAKE2B 6e547d073365cfec6e589f9d0e6227fd75260a96bb07664af472bfbed906662f608b6bf252a42c89de2623a019b83cec5047b7b976ad7d78eb4f3c17866c495a SHA512 8ca7edcd63173f2ba25c786ba1a0ed8f96c93617317cea48a0c76c05614eca4590fb09916cc3e01b0cff4bb924837176b68484a9d50d5cee30622e6b838f5144
--- a/net-proxy/squid-opnsense-fetchacls/files/fetchACLs-github-download.patch
+++ b/net-proxy/squid-opnsense-fetchacls/files/fetchACLs-github-download.patch
@ -0,0 +1,145 @@
+--- squid-opnsense-fetchacls-24.7.12.py	2025-02-03 09:16:20.922603218 +0100
+++ fetchACLs.py	2025-02-03 09:38:51.971220869 +0100
+@@ -54,7 +54,7 @@
+     """ Download helper
+     """
+ 
+-    def __init__(self, url,username, password, timeout, ssl_no_verify=False):
+    def __init__(self, url,username, password, timeout, acl_list, ssl_no_verify=False):
+         """ init new
+             :param url: source url
+             :param timeout: timeout in seconds
+@@ -65,6 +65,7 @@
+         self._username = username
+         self._password = password
+         self._ssl_no_verify = ssl_no_verify
+        self._acl_list = acl_list
+ 
+     def fetch(self):
+         """ fetch (raw) source data into tempfile using self._source_handle
+@@ -115,37 +116,24 @@
+     def get_files(self):
+         """ process downloaded data, handle compression
+             :return: iterator filename, file handle
+            zip-function is written for github-blocklists (ZIP-file including mutiple directories with actual blocklists
+        	e.g. ut1-blocklists-master/blocklists/adult/domains
+         """
+         if self._source_handle is not None:
+-            # handle compressed data
+-            if (len(self._url) > 8 and self._url[-7:] == '.tar.gz') \
+-                    or (len(self._url) > 4 and self._url[-4:] == '.tgz'):
+-                # source is in tar.gz format, extract all into a single string
+-                try:
+-                    tf = tarfile.open(fileobj=self._source_handle)
+-                    for tf_file in tf.getmembers():
+-                        if tf_file.isfile():
+-                            yield tf_file.name, tf.extractfile(tf_file)
+-                except IOError as e:
+-                    syslog.syslog(syslog.LOG_ERR, 'proxy acl: error downloading %s (%s)' % (self._url, e))
+-            elif len(self._url) > 4 and self._url[-3:] == '.gz':
+-                # source is in .gz format unpack
+-                try:
+-                    gf = gzip.GzipFile(mode='r', fileobj=self._source_handle)
+-                    yield os.path.basename(self._url), gf
+-                except IOError as e:
+-                    syslog.syslog(syslog.LOG_ERR, 'proxy acl: error downloading %s (%s)' % (self._url, e))
+-            elif len(self._url) > 5 and self._url[-4:] == '.zip':
+-                # source is in .zip format, extract all into a single string
+-                with zipfile.ZipFile(self._source_handle,
+-                                     mode='r',
+-                                     compression=zipfile.ZIP_DEFLATED) as zf:
+-                    for item in zf.infolist():
+-                        if item.file_size > 0:
+-                            yield item.filename, zf.open(item)
+-            else:
+-                yield os.path.basename(self._url), self._source_handle
+-
+            if len(self._url) > 5 and self._url[-4:] == '.zip':
+                with zipfile.ZipFile(self._source_handle, mode='r') as zip:
+                    for item in zip.infolist():
+                        if item.filename.startswith('ut1-blacklists-master/blacklists/'):
+                            if check_filter(self, item.filename):
+                                if "domains" in item.filename:	#zip contains absolute paths: only if path starts with ...blocklists and ends with domain
+                                    with zip.open(item.filename) as file:	#every domain-file is opened
+                                        print(item.filename)
+                                        content = file.read()
+                                        content = content.decode('utf-8', errors='ignore')	#file-content is parsed into variable and gets decoded to utf-8
+                                        yield item.filename, content
+            elif (len(self._url) > 8 and self._url[-7:] == '.tar.gz') or (len(self._url) > 4 and self._url[-4:] == '.tgz'):
+                tar = tarfile.open(fileobj=self._source_handle)
+                yield from extract_tar(self, tar)
+     def download(self):
+         """ download / unpack ACL
+             :return: iterator filename, type, content
+@@ -154,12 +142,40 @@
+         for filename, filehandle in self.get_files():
+             basefilename = os.path.basename(filename).lower()
+             file_ext = filename.split('.')[-1].lower()
+-            while True:
+-                line = filehandle.readline().decode(encoding='utf-8', errors='ignore')
+-                if not line:
+-                    break
+-                yield filename, basefilename, file_ext, line
+            for line in filehandle.splitlines():
+                line = line.strip()
+                if line:
+                    yield filename, basefilename, file_ext, line
+
+def check_filter(obj, filename):
+    acl_list = obj._acl_list
+    domain = filename.split('/')[-2].lower() #start from end of array and get second last element
+    if len(acl_list) > 0:
+        if domain in acl_list:
+            return True
+        else:
+            return False
+    else:
+        return True
+ 
+def extract_tar(obj, tar_file, parent_dir=''):
+    for tf_file in tar_file.getmembers():
+        file_name = tf_file.name
+        if tf_file.isfile() and (file_name.endswith('.tar.gz') or file_name.endswith('.tgz')):
+            try:
+                inner_file = tar_file.extractfile(tf_file)
+                inner_tar = tarfile.open(fileobj=inner_file)
+                yield from extract_tar(obj, inner_tar, parent_dir + tf_file.name + '/')
+            except Exception as e:
+                syslog.syslog(syslog.LOG_ERR, 'proxy acl: error downloading or extracting tarball: %s (%s)' % (obj._url, e))
+        elif tf_file.isfile() and not tf_file.name.endswith("."):
+            if "domains" in tf_file.name:
+                if check_filter(obj, tf_file.name):
+                    print(tf_file.name)
+                    content = tar_file.extractfile(tf_file).read().decode('utf-8', errors='ignore')
+                    yield tf_file.name, content
+        else:
+            continue
+ 
+ class DomainSorter(object):
+     """ Helper class for building sorted squid domain acl list.
+@@ -320,7 +336,7 @@
+                         sslNoVerify = True
+                     else:
+                         sslNoVerify = False
+-                    acl = Downloader(download_url, download_username, download_password, acl_max_timeout, sslNoVerify)
+                    acl = Downloader(download_url, download_username, download_password, acl_max_timeout, acl_filters, sslNoVerify)
+                     all_filenames = list()
+                     for filename, basefilename, file_ext, line in acl.download():
+                         if filename_in_ignorelist(basefilename, file_ext):
+@@ -338,16 +354,6 @@
+                         if filename not in all_filenames:
+                             all_filenames.append(filename)
+ 
+-                        if len(acl_filters) > 0:
+-                            acl_found = False
+-                            for acl_filter in acl_filters:
+-                                if acl_filter in filename:
+-                                    acl_found = True
+-                                    break
+-                            if not acl_found:
+-                                # skip this acl entry
+-                                continue
+-
+                         if filetype in targets and targets[filetype]['handle'] is None:
+                             targets[filetype]['handle'] = targets[filetype]['class'](targets[filetype]['filename'])
+                         if filetype in targets:
--- a/net-proxy/squid-opnsense-fetchacls/squid-opnsense-fetchacls-24.7.12.ebuild
+++ b/net-proxy/squid-opnsense-fetchacls/squid-opnsense-fetchacls-24.7.12.ebuild
@ -33,13 +33,18 @@ src_prepare() {
 		-e 's|/usr/local/etc/squid/acl|/var/lib/squid/acl|' \
 		${P}.py || die

+	cp ${P}.py fetchACLs.py
+	eapply -p0 "${FILESDIR}/fetchACLs-github-download.patch"
+
 	eapply_user
 }


 src_install() {
-	newbin ${P}.py fetchACLs.py
+	dobin fetchACLs.py
 	python_replicate_script "${D}"/usr/bin/fetchACLs.py
+	newbin ${P}.py fetchACLs-OPNsense.py
+	python_replicate_script "${D}"/usr/bin/fetchACLs-OPNsense.py
 	systemd_dounit "${FILESDIR}"/fetchACLs.{service,timer}
 	insinto /etc/squid
 	doins "${FILESDIR}"/externalACLs.conf "${FILESDIR}"/squid.conf.*