forked from docc-lab/openstack-build-ubuntu
-
Notifications
You must be signed in to change notification settings - Fork 0
/
setup-lib.sh
executable file
·1632 lines (1481 loc) · 48.5 KB
/
setup-lib.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/sh
DIRNAME=`dirname $0`
#
# Setup our core vars
#
OURDIR=/root/setup
SETTINGS=$OURDIR/settings
LOCALSETTINGS=$OURDIR/settings.local
TOPOMAP=$OURDIR/topomap
BOOTDIR=/var/emulab/boot
TMCC=/usr/local/etc/emulab/tmcc
# Setup time logging stuff early
TIMELOGFILE=$OURDIR/setup-time.log
FIRSTTIME=0
if [ ! -f $OURDIR/setup-lib-first ]; then
touch $OURDIR/setup-lib-first
FIRSTTIME=`date +%s`
fi
logtstart() {
area=$1
varea=`echo $area | sed -e 's/[^a-zA-Z_0-9]/_/g'`
stamp=`date +%s`
date=`date`
eval "LOGTIMESTART_$varea=$stamp"
echo "START $area $stamp $date" >> $TIMELOGFILE
}
logtend() {
area=$1
#varea=${area//-/_}
varea=`echo $area | sed -e 's/[^a-zA-Z_0-9]/_/g'`
stamp=`date +%s`
date=`date`
eval "tss=\$LOGTIMESTART_$varea"
tsres=`expr $stamp - $tss`
resmin=`perl -e 'print '"$tsres"' / 60.0 . "\n"'`
echo "END $area $stamp $date" >> $TIMELOGFILE
echo "TOTAL $area $tsres $resmin" >> $TIMELOGFILE
}
if [ $FIRSTTIME -ne 0 ]; then
logtstart "libfirsttime"
fi
mkdir -p $OURDIR
touch $SETTINGS
touch $LOCALSETTINGS
cd $OURDIR
#LOCKFILE="lockfile -1 -r -1 "
LOCKFILE="lockfile-create --retry 65535 "
RMLOCKFILE="lockfile-remove "
PSWDGEN="openssl rand -hex 10"
SSH="ssh -o StrictHostKeyChecking=no"
SCP="scp -p -o StrictHostKeyChecking=no"
#
# Our default configuration
#
CONTROLLER="ctl"
NETWORKMANAGER="nm"
STORAGEHOST="ctl"
SHAREHOST="ctl"
OBJECTHOST="ctl"
COMPUTENODES=""
BAREMETALNODES=""
BLOCKNODES=""
OBJECTNODES=""
DATALAN="lan-1"
MGMTLAN="lan-2"
BLOCKLAN=""
OBJECTLAN=""
DATATUNNELS=1
DATAFLATLANS="lan-1"
DATAVLANS=""
DATAVXLANS=0
DATAOTHERLANS=""
USE_EXISTING_IPS=1
DO_APT_INSTALL=1
DO_APT_UPGRADE=0
DO_APT_DIST_UPGRADE=0
DO_APT_UPDATE=1
UBUNTUMIRRORHOST=""
UBUNTUMIRRORPATH=""
ENABLE_NEW_SERIAL_SUPPORT=0
DO_UBUNTU_CLOUDARCHIVE=0
DO_UBUNTU_CLOUDARCHIVE_STAGING=0
BUILD_AARCH64_FROM_CORE=0
DISABLE_SECURITY_GROUPS=0
ENABLE_HOST_PASSTHROUGH=0
DEFAULT_SECGROUP_ENABLE_SSH_ICMP=1
VERBOSE_LOGGING="False"
DEBUG_LOGGING="False"
SUPPORT_DYNAMIC_NODES=0
KEYSTONEAPIVERSION=""
TOKENTIMEOUT=14400
SESSIONTIMEOUT=14400
GLANCE_LV_SIZE=32
SWIFT_LV_SIZE=4
CEILOMETER_USE_WSGI=0
QUOTASOFF=1
# Off by default; seems to cause intermittent keystone unavailability.
KEYSTONEUSEMEMCACHE=0
# Off by default for Juno; on for Kilo and on by default.
KEYSTONEUSEWSGI=""
# On by default; users will have to take full disk images of the
# compute nodes if they have this enabled.
COMPUTE_EXTRA_NOVA_DISK_SPACE="1"
# Support linuxbridge plugin too, but still default to openvswitch.
ML2PLUGIN="openvswitch"
MANILADRIVER="generic"
EXTRAIMAGEURLS=""
LINUXBRIDGE_STATIC=0
# If set to 1, and if OSRELEASE >= OSNEWTON, the physical machines will
# use the MGMTIP as the primary DNS server (in preference to the real
# control net DNS server). The local domain will also be searched prior
# to the cluster's domain.
USE_DESIGNATE_AS_RESOLVER=1
# If set to 1, and if OSRELEASE >= OSNEWTON, then setup Neutron LBaaS.
USE_NEUTRON_LBAAS=1
# We are not currently using the ceilometer stats, and they do not work
# as of Pike due to the switch to Gnocchi as the measurement DB.
ENABLE_OPENSTACK_SLOTHD=0
# The input OpenStack release, if any, from profile params.
OSRELEASE=""
# If "", no resizing. If set to number, that is GB. Even if you
# specify MB, the MB will be stripped and assumed to be GB. If set to
# 0, the max amount of space after removing swap and extra partitions
# will be used.
RESIZEROOT=""
#
# We have an 'adminapi' user that gets a random password. Then, we have
# the dashboard and instance password, that comes in from geni-lib/rspec as a
# hash, that defaults to the same as the old default profile.
#
# /root/setup/admin-openrc.sh contains the adminapi user, not admin! We do it
# this way because we need a real passwd so that various CLI tools and openstack
# components have a real user/pass to auth as.
#
ADMIN_API='adminapi'
ADMIN_API_PASS=`$PSWDGEN`
ADMIN='admin'
ADMIN_PASS=''
#ADMIN_PASS_HASH='$6$kOIVUcvsnrD/hETx$JahyKoIJf1EFNI2AWCtfzn3ZBoBfaJrRQkjC0kW6VkTwPI9K3TtEWTh/axrHP.e5mmcM96/bTQs1.e7HSKIk10'
ADMIN_PASS_HASH=''
SWAPPER=`cat $BOOTDIR/swapper`
##
## Are we updating?
##
if [ "x$UPDATING" = "x" ]; then
UPDATING=0
elif [ ! $UPDATING -eq 0 ]; then
$LOCKFILE $OURDIR/UPDATING
fi
# We might store any new nodes here
NEWNODELIST=""
# We might store any missing nodes here
OLDNODELIST=""
#
# Setup apt-get to not prompt us
#
if [ ! -e $OURDIR/apt-configured ]; then
echo "force-confdef" > /etc/dpkg/dpkg.cfg.d/cloudlab
echo "force-confold" >> /etc/dpkg/dpkg.cfg.d/cloudlab
touch $OURDIR/apt-configured
fi
export DEBIAN_FRONTEND=noninteractive
# -o Dpkg::Options::="--force-confold" -o Dpkg::Options::="--force-confdef"
DPKGOPTS=''
APTGETINSTALLOPTS='-y'
APTGETINSTALL="apt-get $DPKGOPTS install $APTGETINSTALLOPTS"
# Don't install/upgrade packages if this is not set
if [ ${DO_APT_INSTALL} -eq 0 ]; then
APTGETINSTALL="/bin/true ${APTGETINSTALL}"
fi
##
## Detect if this was a geni experiment
##
grep GENIUSER $SETTINGS
if [ ! $? -eq 0 ]; then
which python
if [ ! $? -eq 0 ]; then
# We need python to continue; install it.
apt-get update
apt-get $DPKGOPTS install $APTGETINSTALLOPTS python
fi
geni-get slice_urn >/dev/null 2>&1
if [ $? -eq 0 ]; then
GENIUSER=1
echo "GENIUSER=1" >> $SETTINGS
else
GENIUSER=0
echo "GENIUSER=0" >> $SETTINGS
fi
else
grep GENIUSER=1 $SETTINGS
if [ $? -eq 0 ]; then
GENIUSER=1
else
GENIUSER=0
fi
fi
##
## Grab our geni creds, and create a GENI credential cert
##
#
# NB: force the install of python-cryptography if geniuser
#
if [ $GENIUSER -eq 1 ]; then
dpkg -s python-cryptography >/dev/null 2>&1
if [ ! $? -eq 0 ]; then
apt-get $DPKGOPTS install $APTGETINSTALLOPTS python-cryptography
# Keep trying again with updated cache forever;
# we must have this package.
success=$?
while [ ! $success -eq 0 ]; do
apt-get update
apt-get $DPKGOPTS install $APTGETINSTALLOPTS python-cryptography
success=$?
done
fi
if [ ! -e $OURDIR/geni.key ]; then
geni-get key > $OURDIR/geni.key
cat $OURDIR/geni.key | grep -q END\ .\*\PRIVATE\ KEY
if [ $? -eq 0 ]; then
HAS_GENI_KEY=1
else
HAS_GENI_KEY=0
fi
else
HAS_GENI_KEY=1
fi
if [ ! -e $OURDIR/geni.certificate ]; then
geni-get certificate > $OURDIR/geni.certificate
cat $OURDIR/geni.certificate | grep -q END\ CERTIFICATE
if [ $? -eq 0 ]; then
HAS_GENI_CERT=1
else
HAS_GENI_CERT=0
fi
else
HAS_GENI_CERT=1
fi
if [ ! -e /root/.ssl/encrypted.pem ]; then
mkdir -p /root/.ssl
chmod 600 /root/.ssl
cat $OURDIR/geni.key > /root/.ssl/encrypted.pem
cat $OURDIR/geni.certificate >> /root/.ssl/encrypted.pem
fi
if [ ! -e $OURDIR/manifests.xml -o $UPDATING -ne 0 ]; then
if [ $HAS_GENI_CERT -eq 1 ]; then
python $DIRNAME/getmanifests.py $OURDIR/manifests
else
# Fall back to geni-get
echo "WARNING: falling back to getting manifest from AM, not Portal -- multi-site experiments will not work fully!"
geni-get manifest > $OURDIR/manifests.0.xml
fi
fi
if [ ! -e $OURDIR/encrypted_admin_pass ]; then
cat /root/setup/manifests.0.xml | perl -e '@lines = <STDIN>; $all = join("",@lines); if ($all =~ /^.+<[^:]+:password[^>]*>([^<]+)<\/[^:]+:password>.+/igs) { print $1; }' > $OURDIR/encrypted_admin_pass
fi
if [ ! -e $OURDIR/decrypted_admin_pass -a -s $OURDIR/encrypted_admin_pass ]; then
openssl smime -decrypt -inform PEM -inkey geni.key -in $OURDIR/encrypted_admin_pass -out $OURDIR/decrypted_admin_pass
fi
fi
#
# Suck in user configuration overrides, if we haven't already
#
if [ ! -e $OURDIR/parameters ]; then
touch $OURDIR/parameters
if [ $GENIUSER -eq 1 ]; then
cat $OURDIR/manifests.0.xml | sed -n -e 's/^[^<]*<[^:]*:parameter>\([^<]*\)<\/[^:]*:parameter>/\1/p' > $OURDIR/parameters
fi
fi
. $OURDIR/parameters
#
# Ok, to be absolutely safe, if the ADMIN_PASS_HASH we got from params was "",
# and if admin pass wasn't sent as an encrypted string to us, we have we have
# to generate a random admin pass and hash it.
#
if [ "x${ADMIN_PASS_HASH}" = "x" ] ; then
DEC_ADMIN_PASS=`cat $OURDIR/decrypted_admin_pass`
if [ "x${DEC_ADMIN_PASS}" = "x" ]; then
ADMIN_PASS=`$PSWDGEN`
ADMIN_PASS_HASH="`echo \"${ADMIN_PASS}\" | openssl passwd -1 -stdin`"
# Save it off so we can email the user -- because nobody has the
# random pass we just generated!
echo "${ADMIN_PASS}" > $OURDIR/random_admin_pass
else
ADMIN_PASS="${DEC_ADMIN_PASS}"
ADMIN_PASS_HASH="`echo \"${ADMIN_PASS}\" | openssl passwd -1 -stdin`"
fi
#
# Overwrite the params.
#
echo "ADMIN_PASS='${ADMIN_PASS}'" >> $OURDIR/parameters
echo "ADMIN_PASS_HASH='${ADMIN_PASS_HASH}'" >> $OURDIR/parameters
fi
CREATOR=`cat $BOOTDIR/creator`
SWAPPER=`cat $BOOTDIR/swapper`
NODEID=`cat $BOOTDIR/nickname | cut -d . -f 1`
PNODEID=`cat $BOOTDIR/nodeid`
EEID=`cat $BOOTDIR/nickname | cut -d . -f 2`
EPID=`cat $BOOTDIR/nickname | cut -d . -f 3`
OURDOMAIN=`cat $BOOTDIR/mydomain`
NFQDN="`cat $BOOTDIR/nickname`.$OURDOMAIN"
PFQDN="`cat $BOOTDIR/nodeid`.$OURDOMAIN"
MYIP=`cat $BOOTDIR/myip`
EXTERNAL_NETWORK_INTERFACE=`cat $BOOTDIR/controlif`
HOSTNAME=`cat ${BOOTDIR}/nickname | cut -f1 -d.`
ARCH=`uname -m`
# Check if our init is systemd
dpkg-query -S /sbin/init | grep -q systemd
HAVE_SYSTEMD=`expr $? = 0`
#
# Figure out which OS/OpenStack this is.
#
OSJUNO=10
OSKILO=11
OSLIBERTY=12
OSMITAKA=13
OSNEWTON=14
OSOCATA=15
OSPIKE=16
OSQUEENS=17
OSROCKY=18
OSSTEIN=19
. /etc/lsb-release
#
# Allow a specific release to trump the image defaults, maybe.
#
if [ ! "x$OSRELEASE" = "x" ]; then
OSCODENAME="$OSRELEASE"
if [ $OSCODENAME = "juno" ]; then OSVERSION=$OSJUNO ; fi
if [ $OSCODENAME = "kilo" ]; then OSVERSION=$OSKILO ; fi
if [ $OSCODENAME = "liberty" ]; then OSVERSION=$OSLIBERTY ; fi
if [ $OSCODENAME = "mitaka" ]; then OSVERSION=$OSMITAKA ; fi
if [ $OSCODENAME = "newton" ]; then OSVERSION=$OSNEWTON ; fi
if [ $OSCODENAME = "ocata" ]; then OSVERSION=$OSOCATA ; fi
if [ $OSCODENAME = "pike" ]; then OSVERSION=$OSPIKE ; fi
if [ $OSCODENAME = "queens" ]; then OSVERSION=$OSQUEENS ; fi
if [ $OSCODENAME = "rocky" ]; then OSVERSION=$OSROCKY ; fi
if [ $OSCODENAME = "stein" ]; then OSVERSION=$OSSTEIN ; fi
#
# We only use cloudarchive for LTS images!
#
echo "$DISTRIB_DESCRIPTION" | grep -qi LTS
if [ $? -eq 0 ]; then
DO_UBUNTU_CLOUDARCHIVE=1
fi
elif [ ${DISTRIB_CODENAME} = "wily" ]; then
OSCODENAME="liberty"
OSVERSION=$OSLIBERTY
elif [ ${DISTRIB_CODENAME} = "vivid" ]; then
OSCODENAME="kilo"
OSVERSION=$OSKILO
elif [ ${DISTRIB_CODENAME} = "xenial" ]; then
OSCODENAME="mitaka"
OSVERSION=$OSMITAKA
else
OSCODENAME="juno"
OSVERSION=$OSJUNO
fi
DISTRIB_MAJOR=`echo $DISTRIB_RELEASE | cut -d. -f1`
#
# Default memcached fully on for Mitaka or greater. Too slow without it.
#
if [ $OSVERSION -ge $OSMITAKA ]; then
KEYSTONEUSEMEMCACHE=1
fi
if [ $OSVERSION -eq $OSJUNO ]; then
REGION="regionOne"
else
REGION="RegionOne"
fi
#
# Stop using auth_uri in favor of www_authenticate_uri at Stein.
#
AUTH_URI_KEY="auth_uri"
if [ $OSVERSION -ge $OSSTEIN ]; then
AUTH_URI_KEY="www_authenticate_uri"
fi
#
# We started using Python 3 packages at Stein.
#
PYPKGPREFIX="python"
ISPYTHON3=0
PYTHONBINNAME="python"
if [ $OSVERSION -ge $OSSTEIN ]; then
PYPKGPREFIX="python3"
ISPYTHON3=1
PYTHONBINNAME="python3"
#
# If our openstack python libs will be python3, we also need
# python3-cryptography for setup-user-info.py, because that file
# must be run by whatever python version contains the novaclient
# library. And we don't know that we need that until here, unlike
# above where we install the v2 python-cryptography.
#
if [ $GENIUSER -eq 1 ]; then
dpkg -s python-cryptography >/dev/null 2>&1
if [ ! $? -eq 0 ]; then
apt-get $DPKGOPTS install $APTGETINSTALLOPTS ${PYTHONBINNAME}-cryptography
# Keep trying again with updated cache forever;
# we must have this package.
success=$?
while [ ! $success -eq 0 ]; do
apt-get update
apt-get $DPKGOPTS install $APTGETINSTALLOPTS ${PYTHONBINNAME}-cryptography
success=$?
done
fi
fi
fi
#
# See which keystone port has the admin capabilities. This changed in
# Queens to fully drop 35357 because it's now irrelevant and not
# configured by default.
#
if [ $OSVERSION -ge $OSQUEENS ]; then
KADMINPORT=5000
else
KADMINPORT=35357
fi
#
# Figure out if we got told to use keystone v2 or v3, or what our
# default should be if not.
#
if [ "x$KEYSTONEAPIVERSION" = "x3" ]; then
# Let them force v3.
KAPISTR='v3'
elif [ "$KEYSTONEAPIVERSION" != "2" -a $OSVERSION -ge $OSLIBERTY ]; then
# If they didn't force v2 or v3, if we're on Liberty or higher, make
# v3 the default
KAPISTR='v3'
KEYSTONEAPIVERSION=3
else
# Otherwise, use version 2 by default (or choice)
KEYSTONEAPIVERSION=2
KAPISTR='v2.0'
fi
#
# Figure out Nova API string.
#
NAPISTR="v2"
if [ $OSVERSION -ge $OSMITAKA ]; then
NAPISTR="v2.1"
fi
#
# Figure out if we got told to use keystone wsgi or not, or what our
# default should be if not.
#
if [ "x$KEYSTONEUSEWSGI" = "x" -a $OSVERSION -ge $OSKILO ]; then
KEYSTONEUSEWSGI=1
elif [ "x$KEYSTONEUSEWSGI" = "x1" ]; then
# Let them force WSGI
KEYSTONEUSEWSGI=1
else
KEYSTONEUSEWSGI=0
fi
#
# The keystone auth_token parameter names are project_domain_name and
# user_domain_name as of Mitaka. The auth_token parameter name has
# also changed.
#
if [ $OSVERSION -ge $OSMITAKA ]; then
PROJECT_DOMAIN_PARAM="project_domain_name"
USER_DOMAIN_PARAM="user_domain_name"
AUTH_TYPE_PARAM="auth_type"
else
PROJECT_DOMAIN_PARAM="project_domain_id"
USER_DOMAIN_PARAM="user_domain_id"
AUTH_TYPE_PARAM="auth_plugin"
fi
#
# Set the database package name and driver string.
#
if [ $OSVERSION -ge $OSNEWTON ]; then
DBDPACKAGE="python-pymysql"
DBDSTRING="mysql+pymysql"
else
DBDPACKAGE="python-mysqldb"
DBDSTRING="mysql"
fi
if [ "x$PROGRESS_EMAIL" = "x" ]; then
if [ $GENIUSER -eq 1 ]; then
SWAPPER_EMAIL=`geni-get slice_email`
else
SWAPPER_EMAIL="$SWAPPER@$OURDOMAIN"
fi
else
SWAPPER_EMAIL="$PROGRESS_EMAIL"
fi
if [ $GENIUSER -eq 1 ]; then
PUBLICADDRS=`cat $OURDIR/manifests.*.xml | perl -e '$found = 0; while (<STDIN>) { if ($_ =~ /\<[\d\w:]*routable_pool [^\>\<]*\/>/) { print STDERR "DEBUG: found empty pool: $_\n"; next; } if ($_ =~ /\<[\d\w:]*routable_pool [^\>]*client_id=['"'"'"]'$NETWORKMANAGER'['"'"'"]/) { $found = 1; print STDERR "DEBUG: found: $_\n" } if ($found) { while ($_ =~ m/\<emulab:ipv4 address="([\d.]+)\" netmask=\"([\d\.]+)\"/g) { print "$1\n"; } } if ($found && $_ =~ /routable_pool\>/) { print STDERR "DEBUG: end found: $_\n"; $found = 0; } }' | xargs`
PUBLICCOUNT=0
for ip in $PUBLICADDRS ; do
PUBLICCOUNT=`expr $PUBLICCOUNT + 1`
done
else
PUBLICADDRS=""
PUBLICCOUNT=0
fi
#
# Grab our topomap so we can see how many nodes we have.
# NB: only safe to use topomap for non-fqdn things.
#
if [ ! -f $TOPOMAP -o $UPDATING -ne 0 ]; then
if [ -f $TOPOMAP ]; then
cp -p $TOPOMAP $TOPOMAP.old
fi
# First try via manifest; fall back to tmcc if necessary (although
# that will break multisite exps with >1 second cluster node(s)).
python2 $DIRNAME/manifest-to-topomap.py $OURDIR/manifests.0.xml > $TOPOMAP
if [ ! $? -eq 0 ]; then
echo "ERROR: could not extract topomap from manifest; aborting to tmcc"
rm -f $TOPOMAP
$TMCC topomap | gunzip > $TOPOMAP
fi
# Filter out blockstore nodes
cat $TOPOMAP | grep -v '^bsnode,' > $TOPOMAP.no.bsnode
mv $TOPOMAP.no.bsnode $TOPOMAP
cat $TOPOMAP | grep -v '^bslink,' > $TOPOMAP.no.bslink
mv $TOPOMAP.no.bslink $TOPOMAP
if [ -f $TOPOMAP.old ]; then
diff -u $TOPOMAP.old $TOPOMAP > $TOPOMAP.diff
#
# NB: this does assume that nodes either leave all the lans, or join
# all the lans. We don't try to distinguish anything else.
#
NEWNODELIST=`cat topomap.diff | sed -n -e 's/^\+\([a-zA-Z0-9\-]*\),.*:.*$/\1/p' | uniq | xargs`
OLDNODELIST=`cat topomap.diff | sed -n -e 's/^\-\([a-zA-Z0-9\-]*\),.*:.*$/\1/p' | uniq | xargs`
# Just remove the fqdn map and let it be recalculated below
rm -f $OURDIR/fqdn.map
rm -f $OURDIR/fqdn.physical.map
fi
fi
#
# Since some of our testbeds only have one experiment interface per node,
# just do this little hack job -- if MGMTLAN was specified, but it's not in
# the topomap, we null it out and use the VPN Management setup instead.
#
if [ ! -z "$MGMTLAN" ] ; then
cat $TOPOMAP | grep "$MGMTLAN"
if [ $? -ne 0 ] ; then
echo "*** Cannot find Management LAN $MGMTLAN ; falling back to VPN!"
MGMTLAN=""
fi
fi
#
# Create a map of node nickname to FQDN (and another one of pnode id to FQDN).
# This supports geni multi-site experiments.
#
if [ \( -s $OURDIR/manifests.xml \) -a \( ! \( -s $OURDIR/fqdn.map \) \) ]; then
cat manifests.xml | tr -d '\n' | sed -e 's/<node /\n<node /g' | sed -n -e "s/^<node [^>]*client_id=['\"]*\([^'\"]*\)['\"].*<host name=['\"]\([^'\"]*\)['\"].*$/\1\t\2/p" > $OURDIR/fqdn.map
# Add a newline if we wrote anything.
if [ -s $OURDIR/fqdn.map ]; then
echo '' >> $OURDIR/fqdn.map
fi
# Filter out any blockstore nodes
# XXX: this strategy doesn't work, because only the NM node makes
# the fqdn.map file. So, just look for bsnode for now.
#BSNODES=`cat /var/emulab/boot/tmcc/storageconfig | sed -n -e 's/^.* HOSTID=\([^ \t]*\) .*$/\1/p' | xargs`
#for bs in $BSNODES ; do
# cat $OURDIR/fqdn.map | grep -v "^${bs}"$'\t' > $OURDIR/fqdn.map.tmp
# mv $OURDIR/fqdn.map.tmp $OURDIR/fqdn.map
#done
# XXX: why doesn't the tab grep work here, sigh...
#cat $OURDIR/fqdn.map | grep -v '^bsnode'$'\t' > $OURDIR/fqdn.map.tmp
cat $OURDIR/fqdn.map | grep -v '^bsnode' > $OURDIR/fqdn.map.tmp
mv $OURDIR/fqdn.map.tmp $OURDIR/fqdn.map
cat $OURDIR/fqdn.map | grep -v '^fw[ \t]*' > $OURDIR/fqdn.map.tmp
mv $OURDIR/fqdn.map.tmp $OURDIR/fqdn.map
cat $OURDIR/fqdn.map | grep -v '^fw-s2[ \t]*' > $OURDIR/fqdn.map.tmp
mv $OURDIR/fqdn.map.tmp $OURDIR/fqdn.map
cat manifests.xml | tr -d '\n' | sed -e 's/<node /\n<node /g' | sed -n -e "s/^<node [^>]*component_id=['\"]*[a-zA-Z0-9:\+\.]*node+\([^'\"]*\)['\"].*<host name=['\"]\([^'\"]*\)['\"].*$/\1\t\2/p" > $OURDIR/fqdn.physical.map
# Add a newline if we wrote anything.
if [ -s $OURDIR/fqdn.physical.map ]; then
echo '' >> $OURDIR/fqdn.physical.map
fi
# Filter out any blockstore nodes
cat $OURDIR/fqdn.physical.map | grep -v '[ \t]bsnode\.' > $OURDIR/fqdn.physical.map.tmp
mv $OURDIR/fqdn.physical.map.tmp $OURDIR/fqdn.physical.map
# Filter out any firewall nodes
cat $OURDIR/fqdn.physical.map | grep -v '[ \t]*fw\.' > $OURDIR/fqdn.physical.map.tmp
mv $OURDIR/fqdn.physical.map.tmp $OURDIR/fqdn.physical.map
cat $OURDIR/fqdn.physical.map | grep -v '[ \t]*fw-s2\.' > $OURDIR/fqdn.physical.map.tmp
mv $OURDIR/fqdn.physical.map.tmp $OURDIR/fqdn.physical.map
fi
#
# Setup the fqdn map the non-geni way if necessary!
#
if [ ! -s $OURDIR/fqdn.map ]; then
TNODES=`cat $TOPOMAP | grep -v '^#' | sed -n -e 's/^\([a-zA-Z0-9\-]*\),.*:.*$/\1/p' | xargs`
FQDNS=""
NODES=""
for n in $TNODES ; do
# Filter out any blockstore nodes
grep -q "HOSTID=$n " /var/emulab/boot/tmcc/storageconfig
if [ $? -eq 0 ] ; then
continue
fi
# Filter out any firewall nodes
if [ "$n" = "fw" -o "$n" = "fw-s2" ]; then
continue
fi
fqdn="$n.$EEID.$EPID.$OURDOMAIN"
FQDNS="${FQDNS} $fqdn"
NODES="${NODES} $n"
/bin/echo -e "$n\t$fqdn" >> $OURDIR/fqdn.map
done
fi
#
# Grab our list of short-name and FQDN nodes. One way or the other, we have
# an fqdn map. First we tried the GENI way; then the old Emulab way with
# topomap.
#
NODES=`cat $OURDIR/fqdn.map | cut -f1 | xargs`
FQDNS=`cat $OURDIR/fqdn.map | cut -f2 | xargs`
if [ -z "${COMPUTENODES}" ]; then
# Figure out which networkmanager (netmgr) and controller (ctrl) names we have
for node in $NODES
do
if [ "$node" = "networkmanager" ]; then
NETWORKMANAGER="networkmanager"
fi
if [ "$node" = "controller" ]; then
# If we were using the controller as storage and object host, then
# keep using it (just use the old name we've detected).
if [ "$STORAGEHOST" = "$CONTROLLER" ]; then
STORAGEHOST="controller"
fi
if [ "$OBJECTHOST" = "$CONTROLLER" ]; then
OBJECTHOST="controller"
fi
CONTROLLER="controller"
fi
done
# Figure out which ones are the compute nodes
for node in $NODES
do
if [ "$node" != "$CONTROLLER" -a "$node" != "$NETWORKMANAGER" \
-a "$node" != "$STORAGEHOST" -a "$node" != "$OBJECTHOST" ]; then
COMPUTENODES="$COMPUTENODES $node"
fi
done
fi
OTHERNODES=""
for node in $NODES
do
[ "$node" = "$NODEID" ] && continue
OTHERNODES="$OTHERNODES $node"
done
# Save the node stuff off to settings
grep CONTROLLER $SETTINGS
if [ ! $? = 0 ]; then
echo "CONTROLLER=\"${CONTROLLER}\"" >> $SETTINGS
echo "NETWORKMANAGER=\"${NETWORKMANAGER}\"" >> $SETTINGS
echo "STORAGEHOST=\"${STORAGEHOST}\"" >> $SETTINGS
echo "OBJECTHOST=\"${OBJECTHOST}\"" >> $SETTINGS
echo "COMPUTENODES=\"${COMPUTENODES}\"" >> $SETTINGS
elif [ $UPDATING -ne 0 ]; then
sed -i -e "s/^\(CONTROLLER=\"[^\"]*\"\)\$/CONTROLLER=\"$CONTROLLER\"/" $SETTINGS
sed -i -e "s/^\(NETWORKMANAGER=\"[^\"]*\"\)\$/NETWORKMANAGER=\"$NETWORKMANAGER\"/" $SETTINGS
sed -i -e "s/^\(STORAGEHOST=\"[^\"]*\"\)\$/STORAGEHOST=\"$STORAGEHOST\"/" $SETTINGS
sed -i -e "s/^\(OBJECTHOST=\"[^\"]*\"\)\$/OBJECTHOST=\"$OBJECTHOST\"/" $SETTINGS
sed -i -e "s/^\(COMPUTENODES=\"[^\"]*\"\)\$/COMPUTENODES=\"$COMPUTENODES\"/" $SETTINGS
fi
#
# 0 (true) if networkmanager node is also the controller; 1 if not.
#
unified() {
if [ "$NETWORKMANAGER" = "$CONTROLLER" ]; then
return 0
else
return 1
fi
}
##
## Setup our Ubuntu package mirror, if necessary.
##
grep MIRRORSETUP $SETTINGS
if [ ! $? -eq 0 ]; then
if [ ! "x${UBUNTUMIRRORHOST}" = "x" ]; then
oldstr='us.archive.ubuntu.com'
newstr="${UBUNTUMIRRORHOST}"
if [ ! "x${UBUNTUMIRRORPATH}" = "x" ]; then
oldstr='us.archive.ubuntu.com/ubuntu'
newstr="${UBUNTUMIRRORHOST}/${UBUNTUMIRRORPATH}"
fi
echo "*** Changing Ubuntu mirror from $oldstr to $newstr ..."
sed -E -i.us.archive.ubuntu.com -e "s|(${oldstr})|$newstr|" /etc/apt/sources.list
fi
echo "MIRRORSETUP=1" >> $SETTINGS
fi
if [ ! -f $OURDIR/apt-updated -a "${DO_APT_UPDATE}" = "1" ]; then
#
# Attempt to handle old EOL releases; so far only need to handle utopic
#
. /etc/lsb-release
grep -q old-releases /etc/apt/sources.list
if [ $? != 0 -a "x${DISTRIB_CODENAME}" = "xutopic" ]; then
sed -i -re 's/([a-z]{2}\.)?archive.ubuntu.com|security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list
fi
apt-get update
touch $OURDIR/apt-updated
fi
are_packages_installed() {
retval=1
while [ ! -z "$1" ] ; do
dpkg -s "$1" >/dev/null 2>&1
if [ ! $? -eq 0 ] ; then
retval=0
fi
shift
done
return $retval
}
maybe_install_packages() {
if [ ! ${DO_APT_UPGRADE} -eq 0 ] ; then
# Just do an install/upgrade to make sure the package(s) are installed
# and upgraded; we want to try to upgrade the package.
$APTGETINSTALL $@
return $?
else
# Ok, check if the package is installed; if it is, don't install.
# Otherwise, install (and maybe upgrade, due to dependency side effects).
# Also, optimize so that we try to install or not install all the
# packages at once if none are installed.
are_packages_installed $@
if [ $? -eq 1 ]; then
return 0
fi
retval=0
while [ ! -z "$1" ] ; do
are_packages_installed $1
if [ $? -eq 0 ]; then
$APTGETINSTALL $1
retval=`expr $retval \| $?`
fi
shift
done
return $retval
fi
}
if [ ! -f $OURDIR/cloudarchive-added -a "${DO_UBUNTU_CLOUDARCHIVE}" = "1" ]; then
#if [ "${DISTRIB_CODENAME}" = "trusty" ] ; then
# $APTGETINSTALL install -y ubuntu-cloud-keyring
# echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu" "${DISTRIB_CODENAME}-updates/${OSCODENAME} main" > /etc/apt/sources.list.d/cloudarchive-${OSCODENAME}.list
# apt-get update
#elif [ "${DISTRIB_CODENAME}" = "xenial" ] ; then
maybe_install_packages software-properties-common
# Disable unattended upgrades!
rm -fv /etc/apt/apt.conf.d/*unattended-upgrades
add-apt-repository -y cloud-archive:$OSRELEASE
if [ "${DO_UBUNTU_CLOUDARCHIVE_STAGING}" = "1" ]; then
add-apt-repository -y cloud-archive:${OSRELEASE}-proposed
fi
apt-get update
#fi
touch $OURDIR/cloudarchive-added
fi
if [ ! -f $OURDIR/apt-dist-upgraded -a "${DO_APT_DIST_UPGRADE}" = "1" ]; then
# First, mark grub packages not to be upgraded; we don't want an
# install going to the wrong place.
PKGS="grub-common grub-gfxpayload-lists grub-pc grub-pc-bin grub2-common"
for pkg in $PKGS; do
apt-mark hold $pkg
done
apt-get dist-upgrade -y
for pkg in $PKGS; do
apt-mark unhold $pkg
done
touch $OURDIR/apt-dist-upgraded
fi
#
# We rely on crudini in a few spots, instead of sed whacking.
#
maybe_install_packages crudini
netmask2prefix() {
nm=$1
bits=0
IFS=.
read -r i1 i2 i3 i4 <<EOF
$nm
EOF
unset IFS
for n in $i1 $i2 $i3 $i4 ; do
v=128
while [ $v -gt 0 ]; do
bits=`expr $bits + \( \( $n / $v \) % 2 \)`
v=`expr $v / 2`
done
done
echo $bits
}
#
# Create IP addresses for the Management and Data networks, as necessary.
#
if [ ! -f $OURDIR/nextsparesubnet ] ; then
#
# This is horrible, but for openstack networks that need IP addrs, that
# were not specified in our experiment description (i.e., by Emulab,
# Cloudlab, or the geni-lib based rspec file), we have to generate them.
# We stay away from 172.16 because Emulab/Cloudlab use it internally for
# virtual machine IP addresses. We also know that Emulab/Cloudlab/geni-lib
# always allocate addresses starting at 10.1.1.1, and increment the second
# octet for a new subnet. We assume that 1) no subnet will ever be larger
# than 255*255 hosts, and 2) that we can start our subnets at 10.254.0.0
# and decrement the second octet any time we need to IP a new openstack
# network.
#
NEXTSPARESUBNET=254
echo ${NEXTSPARESUBNET} > $OURDIR/nextsparesubnet
else
NEXTSPARESUBNET=`cat $OURDIR/nextsparesubnet`
fi
if [ ! -f $OURDIR/mgmt-hosts -o $UPDATING -ne 0 ] ; then
echo "*** Setting up Management and Data Network IP Addresses"
if [ -z "${MGMTLAN}" -o ${USE_EXISTING_IPS} -eq 0 ]; then
if [ $UPDATING -eq 0 ]; then
echo "255.255.0.0" > $OURDIR/mgmt-netmask
echo "192.168.0.1 $NETWORKMANAGER" > $OURDIR/mgmt-hosts
if ! unified ; then
echo "192.168.0.3 $CONTROLLER" >> $OURDIR/mgmt-hosts
fi
o3=0
o4=5
else
o3=`cat $OURDIR/mgmt-o3`
o4=`cat $OURDIR/mgmt-o4`
fi
for node in $NODES
do
[ "$node" = "$CONTROLLER" -o "$node" = "$NETWORKMANAGER" ] \
&& continue
# If it already exists, skip it.
grep -q ${node}\$ $OURDIR/mgmt-hosts
if [ $? -eq 0 ]; then
continue
fi
echo "192.168.$o3.$o4 $node" >> $OURDIR/mgmt-hosts
# Skip 2 for openvpn tun tunnels
o4=`expr $o4 + 2`
if [ $o4 -gt 253 ] ; then
o4=10
o3=`expr $o3 + 1`
fi
done
# Save off our octets for later
echo "$o3" > $OURDIR/mgmt-o3
echo "$o4" > $OURDIR/mgmt-o4
else
cat $TOPOMAP | grep -v '^#' | sed -e 's/,/ /' \
| sed -n -e "s/\([a-zA-Z0-9_\-]*\) .*${MGMTLAN}:\([0-9\.]*\).*\$/\2\t\1/p" \
> $OURDIR/mgmt-hosts
cat ${BOOTDIR}/tmcc/ifconfig \
| sed -n -e "s/^.* MASK=\([0-9\.]*\) .* LAN=${MGMTLAN}.*$/\1/p" \
> $OURDIR/mgmt-netmask
fi
#
# If USE_EXISTING_IPS is set to 0, we will re-IP those data lan
# interfaces: networkmanager:eth1 gets 10.z.0.1/8, and controller gets
# 10.z.0.3/8; and the compute nodes get 10.z.x.y, where x starts at 1,
# and y starts at 1 and does not exceed 254.
#
# We only assign IPs to flat networks because they are the only networks
# that the physical hosts (i.e., controller, networkmanager, compute nodes)
# have an interface on. Technically, we don't *need* to do this, but do it.
# One thing it gives us is the ability to run GRE or VXLAN networks over the
# flat networks.
#
if [ ${USE_EXISTING_IPS} -eq 0 ]; then
for lan in $DATAFLATLANS $DATAVLANS $DATAOTHERLANS ; do
if [ $UPDATING -eq 0 ]; then
prefix="10.$NEXTSPARESUBNET"
echo "$prefix" > $OURDIR/data-prefix.$lan
echo "255.255.0.0" > $OURDIR/data-netmask.$lan
echo "$prefix.0.0/16" > $OURDIR/data-cidr.$lan
echo "$prefix.0.0" > $OURDIR/data-network.$lan
echo "$prefix.0.1 $NETWORKMANAGER" > $OURDIR/data-hosts.$lan
if ! unified ; then
echo "$prefix.0.3 $CONTROLLER" >> $OURDIR/data-hosts.$lan
fi
#
# Now set static IPs for the compute nodes.
#
o3=1
o4=1
else
prefix=`cat $OURDIR/data-prefix.$lan`
o3=`cat $OURDIR/data-o3.$lan`
o4=`cat $OURDIR/data-o4.$lan`
fi
for node in $NODES
do
[ "$node" = "$CONTROLLER" -o "$node" = "$NETWORKMANAGER" ] \
&& continue
# If it already exists, skip it.
grep -q ${node}\$ $OURDIR/data-hosts.$lan
if [ $? -eq 0 ]; then
continue
fi
echo "$prefix.$o3.$o4 $node" >> $OURDIR/data-hosts
# Skip 2 for openvpn tun tunnels
o4=`expr $o4 + 1`
if [ $o4 -gt 254 ] ; then
o4=10
o3=`expr $o3 + 1`
fi
done
if [ $o3 -gt 128 ]; then
echo "ERROR: more physical hosts than $prefix.$o3 ; aborting!"
exit 1
fi
# Save off our octets for later
echo $o3 > $OURDIR/data-o3.$lan