scripts: replace sync-stage3.sh with fetch-stage3

This new script is smarter and more powerful.  The only drawback is that
instead of rsync it uses HTTP.

Makefile

@@ -127,7 +127,7 @@ stage3-$(ARCH)-latest.tar.bz2:
 	@exit 1
 
 sync_stage3:
-	./scripts/sync-stage3.sh $(ARCH)
+	./scripts/fetch-stage3 --specialty=systemd --outfile=stage3-$(ARCH)-latest.tar.bz2 $(ARCH)
 
 
 $(STAGE3): stage3-$(ARCH)-latest.tar.bz2

scripts/fetch-stage3

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""fetch a stage3 from Gentoo's mirror"""
+
+import argparse
+import urllib.request
+import os
+import re
+import sys
+from collections import namedtuple
+
+MIRROR = 'http://gentoo.osuosl.org/'
+Stage3 = namedtuple('Stage3', 'url cpu specialty minimal nomultilib size')
+MINIMAL = re.compile(r'[\+-]minimal-')
+NOMULTILIB = re.compile(r'[\+-]nomultilib-')
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--cpu', type=str, default=None)
+    parser.add_argument('--specialty', type=str, default=None)
+    parser.add_argument('--no-multilib', action='store_true', default=False)
+    parser.add_argument('--minimal', action='store_true', default=False)
+    parser.add_argument('--outfile', type=str, default=None)
+    parser.add_argument('arch', type=str)
+
+    return parser.parse_args()
+
+
+def get_manifest(arch, mirror=MIRROR):
+    manifest = []
+    url = mirror + '/releases/{0}/autobuilds/latest-stage3.txt'.format(arch)
+    base_url = mirror + '/releases/{0}/autobuilds/'.format(arch)
+
+    with urllib.request.urlopen(url) as fp:
+        text = fp.read()
+        text = text.decode('ascii')
+
+    for line in text.split('\n'):
+        line = line.strip()
+
+        if not line or line[0] == '#':
+            continue
+
+        filename, size = line.split()
+        size = int(size)
+        dirname, rest = filename.split('/', 1)
+
+        if '/' in rest:
+            specialty, rest = rest.split('/', 1)
+        else:
+            specialty = None
+
+        cpu = rest.split('-', 2)[1]
+        nomultilib = 'nomultilib-' in rest
+        nomultilib = bool(NOMULTILIB.search(rest))
+        minimal = bool(MINIMAL.search(rest))
+
+        stage3 = Stage3(
+            cpu=cpu,
+            minimal=minimal,
+            nomultilib=nomultilib,
+            size=size,
+            specialty=specialty,
+            url=base_url + filename,
+        )
+
+        manifest.append(stage3)
+
+    return manifest
+
+
+def fetch(url, outfile):
+    """fetch url and write it to outfile"""
+    with urllib.request.urlopen(url) as infile, open(outfile, 'wb') as outfile:
+        while True:
+            data = infile.read(512)
+
+            if not data:
+                break
+
+            outfile.write(data)
+
+
+def main():
+    args = parse_args()
+    manifest = get_manifest(args.arch)
+    manifest.reverse()
+    stage3 = None
+
+    for item in manifest:
+        if args.cpu and item.cpu != args.cpu:
+            continue
+
+        if not all([
+            args.minimal == item.minimal,
+            args.no_multilib == item.nomultilib,
+            args.specialty == item.specialty,
+        ]):
+            continue
+
+        stage3 = item
+        break
+
+    if not stage3:
+        msg = 'No stage3 found matching your criteria\n'
+        sys.stderr.write(msg)
+        sys.exit(404)
+
+    filename = os.path.basename(stage3.url)
+    if args.outfile:
+        outfile = args.outfile
+    else:
+        outfile = filename
+
+    print(filename)
+    fetch(stage3.url, outfile)
+    stat = os.stat(outfile)
+
+    assert stat.st_size == stage3.size, 'File size does not match manifest'
+
+main()

scripts/sync-stage3.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-set -e
-RSYNC_MIRROR=${RSYNC_MIRROR:-rsync://mirrors.rit.edu/gentoo/}
-arch=$1
-g_arch=$(echo ${arch}|sed 's/i.86/x86/')
-rsync="rsync --no-motd"
-echo -n ${arch}:
-
-latest=/releases/${g_arch}/autobuilds/latest-stage3.txt
-
-${rsync} ${RSYNC_MIRROR}${latest} latest-stage3.txt
-file=$(egrep -v 'nomultilib|hardened|uclibc|^#' latest-stage3.txt \
-       | grep -E ${arch}\|i686 |head -n 1 \
-       | cut -d ' ' -f 1)
-
-file=/releases/${g_arch}/autobuilds/${file}
-echo ${file}
-${rsync} ${RSYNC_MIRROR}${file} stage3-${arch}-latest.tar.bz2
-rm -f latest-stage3.txt