scripts: replace sync-stage3.sh with fetch-stage3

This new script is smarter and more powerful.  The only drawback is that
instead of rsync it uses HTTP.
This commit is contained in:
Albert Hopkins 2016-07-23 21:11:49 +00:00
parent 8494820fb3
commit 5d1b56c692
3 changed files with 122 additions and 21 deletions

View File

@ -127,7 +127,7 @@ stage3-$(ARCH)-latest.tar.bz2:
@exit 1 @exit 1
sync_stage3: sync_stage3:
./scripts/sync-stage3.sh $(ARCH) ./scripts/fetch-stage3 --specialty=systemd --outfile=stage3-$(ARCH)-latest.tar.bz2 $(ARCH)
$(STAGE3): stage3-$(ARCH)-latest.tar.bz2 $(STAGE3): stage3-$(ARCH)-latest.tar.bz2

121
scripts/fetch-stage3 Executable file
View File

@ -0,0 +1,121 @@
#!/usr/bin/env python3
"""fetch a stage3 from Gentoo's mirror"""
import argparse
import urllib.request
import os
import re
import sys
from collections import namedtuple
MIRROR = 'http://gentoo.osuosl.org/'
Stage3 = namedtuple('Stage3', 'url cpu specialty minimal nomultilib size')
MINIMAL = re.compile(r'[\+-]minimal-')
NOMULTILIB = re.compile(r'[\+-]nomultilib-')
def parse_args():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--cpu', type=str, default=None)
parser.add_argument('--specialty', type=str, default=None)
parser.add_argument('--no-multilib', action='store_true', default=False)
parser.add_argument('--minimal', action='store_true', default=False)
parser.add_argument('--outfile', type=str, default=None)
parser.add_argument('arch', type=str)
return parser.parse_args()
def get_manifest(arch, mirror=MIRROR):
manifest = []
url = mirror + '/releases/{0}/autobuilds/latest-stage3.txt'.format(arch)
base_url = mirror + '/releases/{0}/autobuilds/'.format(arch)
with urllib.request.urlopen(url) as fp:
text = fp.read()
text = text.decode('ascii')
for line in text.split('\n'):
line = line.strip()
if not line or line[0] == '#':
continue
filename, size = line.split()
size = int(size)
dirname, rest = filename.split('/', 1)
if '/' in rest:
specialty, rest = rest.split('/', 1)
else:
specialty = None
cpu = rest.split('-', 2)[1]
nomultilib = 'nomultilib-' in rest
nomultilib = bool(NOMULTILIB.search(rest))
minimal = bool(MINIMAL.search(rest))
stage3 = Stage3(
cpu=cpu,
minimal=minimal,
nomultilib=nomultilib,
size=size,
specialty=specialty,
url=base_url + filename,
)
manifest.append(stage3)
return manifest
def fetch(url, outfile):
"""fetch url and write it to outfile"""
with urllib.request.urlopen(url) as infile, open(outfile, 'wb') as outfile:
while True:
data = infile.read(512)
if not data:
break
outfile.write(data)
def main():
args = parse_args()
manifest = get_manifest(args.arch)
manifest.reverse()
stage3 = None
for item in manifest:
if args.cpu and item.cpu != args.cpu:
continue
if not all([
args.minimal == item.minimal,
args.no_multilib == item.nomultilib,
args.specialty == item.specialty,
]):
continue
stage3 = item
break
if not stage3:
msg = 'No stage3 found matching your criteria\n'
sys.stderr.write(msg)
sys.exit(404)
filename = os.path.basename(stage3.url)
if args.outfile:
outfile = args.outfile
else:
outfile = filename
print(filename)
fetch(stage3.url, outfile)
stat = os.stat(outfile)
assert stat.st_size == stage3.size, 'File size does not match manifest'
main()

View File

@ -1,20 +0,0 @@
#!/bin/bash
set -e
RSYNC_MIRROR=${RSYNC_MIRROR:-rsync://mirrors.rit.edu/gentoo/}
arch=$1
g_arch=$(echo ${arch}|sed 's/i.86/x86/')
rsync="rsync --no-motd"
echo -n ${arch}:
latest=/releases/${g_arch}/autobuilds/latest-stage3.txt
${rsync} ${RSYNC_MIRROR}${latest} latest-stage3.txt
file=$(egrep -v 'nomultilib|hardened|uclibc|^#' latest-stage3.txt \
| grep -E ${arch}\|i686 |head -n 1 \
| cut -d ' ' -f 1)
file=/releases/${g_arch}/autobuilds/${file}
echo ${file}
${rsync} ${RSYNC_MIRROR}${file} stage3-${arch}-latest.tar.bz2
rm -f latest-stage3.txt