#!/bin/sh
# ~jhs/bin/.sh/distfiles_fetch	by jhs@
# http://www.berklix.com/~jhs/bin/.sh/distfiles_fetch

# See also:
#	/site/domain/this/etc/make.conf.fetch
#	~/bin/.sh/distfiles_cmpd
#	~/bin/.sh/distfiles_dups
#	~/bin/.sh/distfiles_fetch
#	~/bin/.sh/distfiles_mount

LOG=/pub/freebsd/dists/fetch.date

echo "$0 last ran             : `tail -1 $LOG`"
echo "$0 started `date -u +%Y-%m-%dT%H:%M:%SZ`" >> $LOG

echo "Suggestion: run ctms to see if ports/ is recent"
#	/pub/FreeBSD/branches/-current/ports
#		is maintained via ~/public_html/src/bsd/jhs/bin/local/mail
#	/pri/FreeBSD/branches/-current/ports
#		is examined by ~jhs/public_html/src/bsd/jhs/bin/local/ctms
echo "This may takes a week to run."
echo "You may want to setenv IS_INTERACTIVE or BATCH."
#	See /usr/ports/Mk/* re IS_INTERACTIVE & BATCH
#	*/ brackets.c matcher.
#	If a backup of distfiles is running at the same time,
#	it will waste CPU & may overflow disk,
cd /pub/FreeBSD/dists || exit 1

echo "Warning: Consider if you have enough free disk space:"
echo "At 2012_11 dists/current needed 97 Gig."
echo "Current size in K is `du -s -k /pub/FreeBSD/dists/current`"
df `cd /pub/FreeBSD/dists/current;/bin/pwd`

echo "Suggestion: du `pwd`"
echo "Checking ports/ & ports/distfiles."
# PORTSDIRS:
#	Releases for which we have complete ports/
#	directory structures, & will do make fetch.
#	Older ones commented out, now that I run this script fairly often,
#	as it consumes too much CPU, keep refetching cp & cmpd old stuff.
PORTSDIRS="${PORTSDIRS} current"
#	current relies on a link created by: ln -s ../branches/-current current

PORTSDIRS="${PORTSDIRS} 9.3-RELEASE"
PORTSDIRS="${PORTSDIRS} 9.2-RELEASE"
PORTSDIRS="${PORTSDIRS} 6.4-RELEASE"	# Last release for a while with isdn support. ?
PORTSDIRS="${PORTSDIRS} 4.11-RELEASE"

# Add a symbolic link so next for loop can treat current the same as releases.
if test -e /pri/FreeBSD/releases/current ; then	#{
	cd /pri/FreeBSD/releases/current || exit 1
	rm -f ports
	ln -s ../../branches/`uname -m`/-current/ports ports	# usually amd64
	
	if test ! -L /pri/FreeBSD/branches/i386/-current/ports ; then	#{
	  ln -s ../../branches/i386/-current/686/-current/ports \
	    /pri/FreeBSD/branches/i386/-current/ports
	  # At 2015 06 I no longer have /pri/FreeBSD/branches/i386
	  # but I may restore one later.
	fi								#}
fi							#}

# Check all directories exist else exit.
for i in ${PORTSDIRS}
	do	#{
	echo "Starting to check existence of $i {"
	echo "Ensuring existence of /pri/FreeBSD/releases/$i/ports"
	cd /pri/FreeBSD/releases/$i/ports || exit 1
	echo "Ensuring existence of /pri/FreeBSD/releases/$i/ports/distfiles"
	#	Putting this in backets	(cd distfiles || exit 1)
	#	fails to catch error code on non existences,
	#	so as distfiles is a symbolic link elsewhere, add a cd after.
	cd distfiles || exit 1
	cd /pri/FreeBSD/releases/$i/ports || exit 1
	echo "Ensuring /pri/FreeBSD/releases/$i/ports/.customise"
	#	else if not customised, fetch may lock up.
	#	however if it Is customised then ctm update will
	#	periodically fail. Quandary !
	if test -f .customise ; then	#{
		echo "/pri/FreeBSD/releases/$i/ports is customised."
	else							#}{
		echo "/pri/FreeBSD/releases/$i/ports is not customised."
		echo -n "Suggestion: "
		echo "customise /pri/FreeBSD/releases/$i/ports"
		if [ "$i" = current ]; then
		 echo "	For current, specify most recent release, eg:"
		 echo "	customise /pri/FreeBSD/releases/$i/ports `uname -r` "
		fi
	fi							#}
	echo "Ending check of existence of $i }"
	done	#}

echo "Fetching non interactive for all releases"
# Fetch each release.
#	No need to add code to fetch in reverse numeric order of releases,
#	(using something like	echo ${PORTSDIRS} | sed -e 's/ /\n/' | sort -r )
#	for 2 reasons:
#	- Reverse order comes from top of this script.
#	- Release numeric order is not chronological order,
#	 (as we often have 2 major releases interleaving the minor numbers)
for i in ${PORTSDIRS}
	do	#{
	echo "Starting fetch on $i {"
	cd /pri/FreeBSD/releases/$i/ports || exit 1
	/bin/pwd
	PORTSDIR=`pwd` ; export PORTSDIR
	echo "Starting fetch with BATCH on $i {"
	nice make -k BATCH=YES checksum
	#  No -j if debugging
	# -j 2	otherwise single slow sites will delay the whole fetch.
	# -j 3	is too much load
	# "checksum" does more than "fetch", detecting occasional bad
	# distfiles (from eg when make fetch is run in
	# background & DSL link drops periodicaly).
	echo "Ending fetch with Batch on $i }"
	echo "Starting to strip $i {"
	nice distfiles_cmpd
	# The line above can be un-necessarily heavy,
	# comparing & deleting over 30G of largely similar files between
	# latest releas & current. Its best if fetch order here matches strip
	# order in distfiles_cmpd.
	echo "Ending strip of $i }"
	echo "Ending fetch on $i }"
	done	#}

echo "This only fetched distfiles needed by BATCH, not for IS_INTERACTIVE"
echo "so some distfiles needed for interactive current may be languishing in"
echo "/pub/freebsd/dists/current.old , so do not just delete them !"

#echo "Fetching interactive ports, Warning: May hang on input."
#for i in ${PORTSDIRS}
#	do	#{
#	echo "$0 Loop Fetch Start $i {"
#	cd /pri/FreeBSD/releases/$i/ports || exit 1
#	/bin/pwd
#	PORTSDIR=`pwd` ; export PORTSDIR
#	echo "$0 Loop Fetch Interactive Start $i {"
#	echo "Interactive" # More complete, but slower.
#	nice make -k INTERACTIVE=YES checksum
#		# Above, no -j as it would get confusing.
#	echo "$0 Loop Fetch Interactive End $i }"
#	echo "$0 Loop Strip Start $i {"
#	nice distfiles_cmpd
#	echo "$0 Loop Strip End $i }"
#	echo "$0 Loop Fetch End $i }"
#	done	#}

echo "Suggestion: Run nice distfiles_cmpd |& grep Different"

echo "$0 completed `date -u +%Y-%m-%dT%H:%M:%SZ`" >> $LOG