An impossibly large refactor

.circleci/config.yml

@@ -20,9 +20,8 @@
       - image: circleci/ruby:2.4
 
 jobs:
-  build-test:
+  build-test-22:
     executor: python-executor
-    parallelism: 1
     steps:
       - checkout
       - setup_remote_docker
@@ -39,8 +38,104 @@
            /usr/local/bin/cpplint --filter=-whitespace/line_length src/*
            true
       - run:
-         name: Build the image
-         command: bash scripts/docker-outer.sh
+         name: Build the packages
+         command: BUILD_VERSIONS=22 bash scripts/docker-outer.sh
+      - *persist_to_workspace
+
+  build-test-28:
+    executor: python-executor
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Install cpplint
+          command: sudo pip install cpplint
+      - run:
+          name: Set up version
+          command: bash scripts/version.sh
+      - run:
+         name: Call cpplint; expected to fail, so set things up to pass
+         command: |
+           set +eo pipefail
+           /usr/local/bin/cpplint --filter=-whitespace/line_length src/*
+           true
+      - run:
+         name: Build the packages
+         command: BUILD_VERSIONS=28 bash scripts/docker-outer.sh
+      - *persist_to_workspace
+
+  build-test-22-devel:
+    executor: python-executor
+    steps:
+      - checkout
+      - setup_remote_docker
+      - add_ssh_keys:
+          fingerprints:
+            - "17:da:1d:21:58:46:74:fc:38:4a:62:2a:32:79:1d:ab"
+      - run:
+          name: Install cpplint
+          command: sudo pip install cpplint
+      - run:
+          name: Set up version
+          command: bash scripts/version.sh
+      - run:
+         name: Call cpplint; expected to fail, so set things up to pass
+         command: |
+           set +eo pipefail
+           /usr/local/bin/cpplint --filter=-whitespace/line_length src/*
+           true
+      - run:
+         name: Build the packages
+         command: BUILD_VERSIONS=22 DEVEL=-devel bash scripts/docker-outer.sh
+      - run:
+          name: git config
+          command: |
+            git config --global user.email "[email protected]"
+            git config --global user.name "$CIRCLE_USERNAME"
+      - run:
+          name: trigger integration
+          command: |
+            git clone [email protected]:pantheon-systems/fusedav-kube
+            cd fusedav-kube
+            git tag f22-${CIRCLE_BUILD_NUM}-devel
+            git push origin f22-${CIRCLE_BUILD_NUM}-devel
+      - *persist_to_workspace
+
+  build-test-28-devel:
+    executor: python-executor
+    steps:
+      - checkout
+      - setup_remote_docker
+      - add_ssh_keys:
+          fingerprints:
+            - "17:da:1d:21:58:46:74:fc:38:4a:62:2a:32:79:1d:ab"
+      - run:
+          name: Install cpplint
+          command: sudo pip install cpplint
+      - run:
+          name: Set up version
+          command: bash scripts/version.sh
+      - run:
+         name: Call cpplint; expected to fail, so set things up to pass
+         command: |
+           set +eo pipefail
+           /usr/local/bin/cpplint --filter=-whitespace/line_length src/*
+           true
+      - run:
+         name: Build the packages
+         command: BUILD_VERSIONS=28 DEVEL=-devel bash scripts/docker-outer.sh
+      - run:
+          name: git config
+          command: |
+            git config --global user.email "[email protected]"
+            git config --global user.name "$CIRCLE_USERNAME"
+      - run:
+          name: trigger integration
+          command: |
+            git clone [email protected]:pantheon-systems/fusedav-kube
+            cd fusedav-kube
+            git tag f28-${CIRCLE_BUILD_NUM}-devel-${CIRCLE_PULL_REQUEST##*/}
+            git push origin f28-${CIRCLE_BUILD_NUM}-devel-${CIRCLE_PULL_REQUEST##*/}
       - *persist_to_workspace
 
   deploy-dev:
@@ -71,10 +166,28 @@
   version: 2
   build-test-deploy:
     jobs:
-      - build-test
+      - build-test-22
+      - build-test-28
+      - build-test-22-devel:
+          filters:
+            branches:
+              ignore:
+                - dev
+                - yolo
+                - stage
+                - master
+      - build-test-28-devel:
+          filters:
+            branches:
+              ignore:
+                - dev
+                - yolo
+                - stage
+                - master
       - deploy-dev:
           requires:
-            - build-test
+            - build-test-22
+            - build-test-28
           filters:
             branches:
               only:
@@ -83,7 +196,8 @@
                 - stage
       - deploy-prod:
           requires:
-            - build-test
+            - build-test-22
+            - build-test-28
           filters:
             branches:
               only:

Dockerfile

@@ -0,0 +1,8 @@
+ARG VERSION
+
+FROM quay.io/getpantheon/fedora:${VERSION}
+
+COPY pkg/fusedav /opt/fusedav
+RUN dnf install -y /opt/fusedav/*.rpm make perf valgrind gdb \
+  && rm -r /opt/fusedav \
+  && dnf clean all

scripts/docker-inner.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 #
 set -ex
@@ -14,6 +14,11 @@
 build=$3
 epoch=$4
 
+devel=
+if [[ "$fusedav_channel" == *-devel ]]; then
+    devel=true
+fi
+
 fedora_release=$(rpm -q --queryformat '%{VERSION}\n' fedora-release)
 GITSHA=$(git log -1 --format="%h")
 name="fusedav-$fusedav_channel"
@@ -44,6 +49,13 @@
 fi
 
 ./autogen.sh
+SANITIZER_FLAGS=
+if [[ -n $devel ]] && [[ $fusedav_channel == *f28* ]]; then
+    SANITIZER_FLAGS="-fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer -g"
+    yum install -y libasan libubsan
+    export 'LSAN_OPTIONS=verbosity=1:log_threads=1'
+fi
+export SANITIZER_FLAGS
 CURL_LIBS="-lcurl" ./configure
 
 make
@@ -64,10 +76,31 @@
 fi
 set -e
 
+# pack in sources and tests into the rpm
+if [[ -n $devel ]]; then
+    cp -r -t $install_prefix src tests
+
+    iozone_version=iozone3_487
+    curl http://www.iozone.org/src/current/${iozone_version}.tar > ${iozone_version}.tar
+    sha256sum -c sha256sum
+
+    tar xf ${iozone_version}.tar
+
+    pushd 2>&1 ${iozone_version}/src/current > /dev/null
+    make linux-AMD64
+    mv iozone $install_prefix/iozone
+    popd 2>&1 > /dev/null
+fi
+
 mv /usr/local/bin/fusedav $install_prefix/$name
 cp $bin/exec_wrapper/mount.fusedav_chan /usr/sbin/mount.$name
 chmod 755 /usr/sbin/mount.$name
 
+DEP_DEVEL=
+if [[ -n $devel ]]; then
+    DEP_DEVEL="--depends gcc --depends libasan --depends libubsan"
+fi
+
 fpm -s dir -t rpm \
   --name "${name}" \
   --version "${version}" \
@@ -76,9 +109,11 @@
   --url "${url}" \
   --vendor "${vendor}" \
   --description "${description}" \
-  --depends  uriparser \
+  --depends uriparser \
   --depends fuse-libs \
   --depends leveldb \
+  --depends jemalloc \
+  ${DEP_DEVEL} \
   --log=debug \
   $install_prefix \
   /usr/sbin/mount.$name

scripts/docker-outer.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -e
 bin="$(cd -P -- "$(dirname -- "$0")" && pwd -P)"
 docker=$(which docker)
@@ -13,6 +13,8 @@ RUN_ARGS="--rm"
 # set a default build -> 0 for when it doesn't exist
 CIRCLE_BUILD_NUM=${CIRCLE_BUILD_NUM:-0}
 
+DEVEL=${DEVEL:-}
+
 # location to mount the source in the container
 inner_mount="/fusedav"
 
@@ -21,6 +23,8 @@ $docker volume create fusedav_vol
 $docker run --name cp-vol -v fusedav_vol:/fusedav busybox true
 $docker cp $bin/../. cp-vol:/fusedav/
 
+docker login -p "$QUAY_PASSWD" -u "$QUAY_USER" quay.io
+
 # epoch to use for -revision
 epoch=$(date +%s)
 
@@ -31,7 +35,7 @@ for ver in $BUILD_VERSIONS; do
     $docker pull $build_image
 
     channel=$(tr -d "\n\r" < $bin/../CHANNEL)
-    exec_cmd="$inner_mount/scripts/docker-inner.sh $channel $inner_mount/pkg $CIRCLE_BUILD_NUM $epoch"
+    exec_cmd="$inner_mount/scripts/docker-inner.sh $channel$DEVEL $inner_mount/pkg $CIRCLE_BUILD_NUM $epoch"
     if [ -n "$BUILD_DEBUG" ] ; then
       RUN_ARGS="$RUN_ARGS -ti "
       exec_cmd="/bin/bash"
@@ -48,8 +52,21 @@ EOL
     echo "Running: $docker_cmd"
     $docker_cmd
 
+    echo "copying the rpm from the container..."
+    mkdir -p "$bin/../pkg"
+    $docker cp "cp-vol:/fusedav/pkg/${ver}/fusedav" "$bin/../pkg/fusedav"
+    docker_tag="quay.io/getpantheon/fusedav:f${ver}-${CIRCLE_BUILD_NUM}${DEVEL}"
+    docker_build="$docker build -t $docker_tag --build-arg VERSION=${ver} ."
+
+    echo "Running: $docker_build"
+    $docker_build
+
+    docker_push="$docker push $docker_tag"
+
+    echo "Running: $docker_push"
+    $docker_push
+
 done
 
-$docker cp cp-vol:/fusedav/pkg $bin/../pkg/
 $docker rm cp-vol
 $docker volume rm fusedav_vol

sha256sum

@@ -0,0 +1 @@
+2c488a7ccddd624fd557af16e71442c367b131d6178e1b4023bbd532bacdda59  iozone3_487.tar

src/Makefile.am

@@ -30,5 +30,5 @@ fusedav_SOURCES=fusedav.c fusedav.h \
 				stats.c stats.h \
 				fusedav-statsd.c fusedav-statsd.h
 
-fusedav_CFLAGS = $(AM_CFLAGS) $(CURL_CFLAGS) $(URIPARSER_CFLAGS) $(FUSE_CFLAGS) $(YAML_CFLAGS) $(LEVELDB_CFLAGS) $(SYSTEMD_CFLAGS) $(ZLIB_CFLAGS) $(GLIB_CFLAGS) -DFUSE_USE_VERSION=26 -DINJECT_ERRORS=${INJECT_ERRORS}
+fusedav_CFLAGS = $(AM_CFLAGS) $(CURL_CFLAGS) $(URIPARSER_CFLAGS) $(FUSE_CFLAGS) $(YAML_CFLAGS) $(LEVELDB_CFLAGS) $(SYSTEMD_CFLAGS) $(ZLIB_CFLAGS) $(GLIB_CFLAGS) -DFUSE_USE_VERSION=26 -DINJECT_ERRORS=${INJECT_ERRORS} $(SANITIZER_FLAGS)
 fusedav_LDADD = -lpthread -ljemalloc -lrt -lresolv -lexpat $(CURL_LIBS) $(URIPARSER_LIBS) $(FUSE_LIBS) $(YAML_LIBS) $(LEVELDB_LIBS) $(SYSTEMD_LIBS) $(ZLIB_LIBS) $(GLIB_LIBS)