-
Notifications
You must be signed in to change notification settings - Fork 245
/
cvmfsInstall.sh
executable file
·194 lines (183 loc) · 7.12 KB
/
cvmfsInstall.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
#!/bin/sh -ex
source $(dirname $0)/cmsrep.sh
CMS_BOT_DIR=$(dirname $(realpath $0))
source ${CMS_BOT_DIR}/cvmfs_deployment/utils.sh
CVMFS_INSTALL=false
[ "${BASEDIR}" != "" ] || BASEDIR="${CVMFS_BASEDIR}"
case ${BASEDIR} in
/cvmfs/* ) CVMFS_INSTALL=true ;;
esac
${CVMFS_INSTALL} || export USE_CVMFS_GW="false"
export BASEDIR
export THISDIR=$(/bin/pwd -P)
export LC_ALL=C
export LC_CTYPE=C
export LANG=C
ARCHITECTURE=$1
CMS_WEEK=$2
RELEASE_NAME=$3
WORKSPACE=$4
DEV=$5
PROOTDIR=$6
TEST_INSTALL=$7
NUM_WEEKS=$8
REINSTALL_COMMON=$9
INSTALL_PACKAGES="${10}"
CVMFS_PUBLISH_PATH=""
USE_DEV=""
export PROOTDIR
COMMON_BASEDIR="${BASEDIR}"
${CVMFS_INSTALL} && COMMON_BASEDIR="${CVMFS_BASEDIR}"
[ "$PROOTDIR" != "" ] || PROOTDIR=${CVMFS_BASEDIR}/proot
if ${USE_CVMFS_GW} ; then
CMS_ARCH=$(echo ${ARCHITECTURE} | cut -d_ -f2)
[ "${CMS_ARCH}" = "amd64" ] && CMS_ARCH="x86_64"
CVMFS_PUBLISH_PATH="/sw/${CMS_ARCH}"
export BASEDIR="${BASEDIR}${CVMFS_PUBLISH_PATH}"
fi
if [ "$REINSTALL_COMMON" = "true" ] ; then
REINSTALL_COMMON="--reinstall"
else
REINSTALL_COMMON=""
fi
[ "X$NUM_WEEKS" != "X" ] || NUM_WEEKS=2
if [ "X$DEV" = "Xtrue" ] ; then
DEV="-dev"
USE_DEV="--use-dev"
else
DEV=""
fi
cd ${CMS_BOT_DIR}
[ -f ib-weeks ] || exit 1
# The repositories we need to install are those for which we find the
# timestamp files:
REPOSITORIES=`tail -${NUM_WEEKS} ib-weeks | sed -e's/-\([0-9]\)$/-0\1/' | sort -r`
echo $REPOSITORIES
$CVMFS_INSTALL && cvmfs_transaction ${CVMFS_PUBLISH_PATH}
hostname > $BASEDIR/stratum0
#Recreate the links
PUBLISH_CLEANUP=false
for link in $(find $BASEDIR -mindepth 1 -maxdepth 1 -name 'week*' -type l); do unlink $link; done
for t in nweek- ; do
for w in $(find $BASEDIR -mindepth 1 -maxdepth 1 -name "$t*" -type d | sed 's|.*/||') ; do
if [ $(echo "$REPOSITORIES" | grep "^$w$" | wc -l) -gt 0 ] ; then
let N="$(echo $w | cut -d- -f2 | sed 's|^0||') % ${NUM_WEEKS}" || true
ln -s $BASEDIR/$w $BASEDIR/week$N
else
echo "Deleting obsolete week $w"
rm -rf $BASEDIR/$w
$CVMFS_INSTALL && PUBLISH_CLEANUP=true
fi
done
done
if $PUBLISH_CLEANUP ; then
time cvmfs_server publish
cvmfs_transaction ${CVMFS_PUBLISH_PATH}
fi
# We install packages for both weeks. We reset every two week, alternating.
TMP_PREFIX=/tmp/cvsmfs-$$
export SCRAM_ARCH="$ARCHITECTURE"
export CMSPKG_OS_COMMAND="source ${CMS_BOT_DIR}/dockerrun.sh ; dockerrun"
for REPOSITORY in $REPOSITORIES; do
echo $REPOSITORY
let WEEK="$(echo $REPOSITORY | cut -d- -f2 | sed 's|^0*||') % ${NUM_WEEKS}" || true
#If CMS_WEEK was set then only check releases for that week
if [ "X$CMS_WEEK" != "X" -a "$CMS_WEEK" != "cms.week$WEEK" ] ; then
echo "Skipping week for $REPOSITORY"
continue
fi
echo "Checking week $REPOSITORY ($WEEK) for RPMS"
if [ "X$TEST_INSTALL" = "XYes" ] ; then REPOSITORY="test" ; fi
WORKDIR=$BASEDIR/$REPOSITORY
rm -rf $WORKDIR/*/var/cmspkg/rpms || true
mkdir -p $WORKDIR/logs
# Install all architectures of the most recent week first.
CMSPKG="$WORKDIR/common/cmspkg -a $SCRAM_ARCH ${USE_DEV}"
LOGFILE=$WORKDIR/logs/bootstrap-$REPOSITORY-$SCRAM_ARCH.log
#Recover from bad bootstrap arch
if [ -f $LOGFILE -a ! -f $WORKDIR/$SCRAM_ARCH/cms/cms-common/1.0/etc/profile.d/init.sh ] ; then
rm -f $LOGFILE
fi
# If the bootstrap log for the current week is not their rebootstrap the area.
XPKGS="gcc-fixincludes"
if [ ! -f $LOGFILE ]; then
rm -rf $WORKDIR/$SCRAM_ARCH
rm -rf $WORKDIR/bootstraptmp
wget --tries=5 --waitretry=60 -O $WORKDIR/bootstrap.sh http://${CMSREP_IB_SERVER}/cmssw/repos/bootstrap${DEV}.sh
rm -f ${LOGFILE}.err
(source ${CMS_BOT_DIR}/dockerrun.sh ; export CMSPKG_OS_COMMAND="" ; dockerrun "sh -ex $WORKDIR/bootstrap.sh setup ${DEV} -server ${CMSREP_IB_SERVER} -path $WORKDIR -r cms.week$WEEK -arch $SCRAM_ARCH -y >$LOGFILE 2>&1" || touch ${LOGFILE}.err)
if [ -e ${LOGFILE}.err ] ; then
rm -f ${LOGFILE}.err
cat ${LOGFILE}
exit 1
fi
XPKGS="${XPKGS} SCRAMV1 SCRAMV2 cmssw-wm-tools cms-git-tools crab"
[ "${RELEASE_NAME}" != "" ] || XPKGS="${XPKGS} cmssw-tool-conf"
elif [ $(grep "server *${CMSREP_IB_SERVER} " $WORKDIR/common/cmspkg | wc -l) -eq 0 ] ; then
sed -i -e "s| \-\-server *[^ ]* | --server ${CMSREP_IB_SERVER} |" $WORKDIR/common/cmspkg
fi
#Ugrade cmspkg itself
$CMSPKG -y upgrade
#Upgrade any common packages e.g. cms-common, fakesystem etc.
$CMSPKG -y --upgrade-packages upgrade
for pkg in ${XPKGS} ; do
INSTALL_PACKAGES="$(${CMSPKG} --build-order search +${pkg}+ | grep ${pkg} | sed 's| .*||' | head -1) ${INSTALL_PACKAGES}"
done
ln -sfT ${COMMON_BASEDIR}/SITECONF $WORKDIR/SITECONF
if [ $(ls -rtd $WORKDIR/${SCRAM_ARCH}/external/rpm/4.* | tail -1 | sed 's|.*/external/rpm/4.||;s|\..*||') -lt 15 ] ; then
RPM_CONFIG=$WORKDIR/${SCRAM_ARCH}/var/lib/rpm/DB_CONFIG
if [ ! -e $RPM_CONFIG ] ; then
echo "mutex_set_max 10000000" > $RPM_CONFIG
$CMSPKG rpmenv -- rpmdb --rebuilddb
fi
fi
(
[ "${INSTALL_PACKAGES}" = "" ] || ${CMSPKG} -f install ${INSTALL_PACKAGES}
if [ "X$RELEASE_NAME" != "X" ] ; then
x="cms+cmssw-ib+$RELEASE_NAME"
${CMSPKG} clean
${CMSPKG} install -y $x || true
time ${CMSPKG} install --ignore-size -y `echo $x | sed -e 's/cmssw-ib/cmssw/'` || true
time ${CMSPKG} install --ignore-size -y `echo $x | sed -e 's/cmssw-ib/cmssw-patch/'` || true
relname=`echo $x | awk -F + '{print $NF}'`
timestamp=`echo $relname | awk -F _ '{print $NF}' | grep '^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9][0-9][0-9]$' | sed 's|-||g'`
if [ "X$timestamp" != "X" ] ; then
for y in cmssw cmssw-patch ; do
if [ -d $WORKDIR/$SCRAM_ARCH/cms/$y/$relname ] ; then
touch -t $timestamp $WORKDIR/$SCRAM_ARCH/cms/$y/$relname
fi
done
fi
fi
) || true
rm -rf $WORKDIR/*/var/cmspkg/rpms || true
done #End week repository
mkdir -p $BASEDIR/scramdb/etc/scramrc
rm -f $BASEDIR/scramdb/etc/scramrc/links.db
touch $BASEDIR/scramdb/etc/scramrc/links.db
for (( i=0; i<$NUM_WEEKS; i++ )) ; do
echo "$BASEDIR/week$i" >> $BASEDIR/scramdb/etc/scramrc/links.db
done
echo "/cvmfs/cms.cern.ch" >> $BASEDIR/scramdb/etc/scramrc/links.db
#Recreate the links
for link in $(find $BASEDIR -mindepth 1 -maxdepth 1 -name 'week*' -type l); do unlink $link; done
for t in nweek- ; do
for w in $(find $BASEDIR -mindepth 1 -maxdepth 1 -name "$t*" -type d | sed 's|.*/||') ; do
let N="$(echo $w | cut -d- -f2 | sed 's|^0||') % ${NUM_WEEKS}" || true
if [ $(echo "$REPOSITORIES" | grep "^$w$" | wc -l) -gt 0 ] ; then
ln -s $BASEDIR/$w $BASEDIR/week$N
[ -f $BASEDIR/week$N/etc/scramrc/links.db -a -s $BASEDIR/week$N/etc/scramrc/links.db ] && continue
echo "$BASEDIR/scramdb" > $BASEDIR/week$N/etc/scramrc/links.db
else
echo "Deleting obsolete week $w"
rm -rf $BASEDIR/$w
fi
done
done
rm -f $BASEDIR/latest
ln -s $(grep "^nweek-" ${CMS_BOT_DIR}/ib-weeks | tail -1) $BASEDIR/latest
if $CVMFS_INSTALL ; then
# Write everything in the repository
echo "Publishing started" `date`
time cvmfs_server publish
fi