First attempt on cleaning up evaluation

Author: nvasilakis

evaluation/aliases/err.txt evaluation/benchmarks/aliases/err.txt

@@ -1,4 +1,4 @@
 ./oasa-2021-01-08.bz2
-./input.csv
+./in.csv
 ./out
 ./out1

evaluation/aliases/likely-longest-pipelines.txt evaluation/benchmarks/aliases/likely-longest-pipelines.txt

@@ -1,9 +1,6 @@
 #!/bin/bash
-# This script is part of a study on OASA's Telematics
-# Diomidis Spinellis and Eleftheria Tsaliki
-# https://insidestory.gr/article/noymera-leoforeia-athinas
+# Vehicles on the road per day
 
-# # Vehicles on the road per day
 # <in.csv sed 's/T..:..:..//' |
 # awk -F, '!seen[$1 $3] {onroad[$1]++; seen[$1 $3] = 1}
 #    END { OFS = "\t"; for (d in onroad) print d, onroad[d]}' |

evaluation/aliases/process-aliases.sh evaluation/benchmarks/aliases/process-aliases.sh

@@ -1,9 +1,6 @@
 #!/bin/bash
-# This script is part of a study on OASA's Telematics
-# Diomidis Spinellis and Eleftheria Tsaliki
-# https://insidestory.gr/article/noymera-leoforeia-athinas
+# Days a vehicle is on the road
 
-# # Days a vehicle is on the road
 # <in.csv sed 's/T..:..:..//' |
 # awk -F, '!seen[$1 $3] {onroad[$3]++; seen[$1 $3] = 1}
 #    END { OFS = "\t"; for (d in onroad) print d, onroad[d]}' |

evaluation/aliases/run_alias.sh evaluation/benchmarks/aliases/run_alias.sh

@@ -1,9 +1,6 @@
 #!/bin/bash
-# This script is part of a study on OASA's Telematics
-# Diomidis Spinellis and Eleftheria Tsaliki
-# https://insidestory.gr/article/noymera-leoforeia-athinas
+# Hours each vehicle is on the road
 
-# # Hours each vehicle is on the road
 # <in.csv sed 's/T\(..\):..:../,\1/' |
 # awk -F, '!seen[$1 $2 $4] {onroad[$4]++; seen[$1 $2 $4] = 1}
 #    END { OFS = "\t"; for (d in onroad) print d, onroad[d]}' |

evaluation/aliases/setup.sh evaluation/benchmarks/aliases/setup.sh

@@ -1,9 +1,6 @@
 #!/bin/bash
-# This script is part of a study on OASA's Telematics
-# Diomidis Spinellis and Eleftheria Tsaliki
-# https://insidestory.gr/article/noymera-leoforeia-athinas
+# Hours monitored each day
 
-# # Hours monitored each day
 # <in.csv sed 's/T\(..\):..:../,\1/' |
 # awk -F, '!seen[$1 $2] {hours[$1]++; seen[$1 $2] = 1}
 #    END { OFS = "\t"; for (d in hours) print d, hours[d]}' | 

evaluation/aliases/succ.txt evaluation/benchmarks/aliases/succ.txt

@@ -1,8 +1,4 @@
 #!/bin/bash
-# This script is part of a study on OASA's Telematics
-# Diomidis Spinellis and Eleftheria Tsaliki
-# https://insidestory.gr/article/noymera-leoforeia-athinas
-
 # Hours each bus is active each day
 
 # Records are day, hour, line, bus

evaluation/log_parsing/nginx_scripts/nginx.sh evaluation/benchmarks/analysis-logs/nginx.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# curl https://www.balab.aueb.gr/~dds/oasa-$(date --date='1 days ago' +'%y-%m-%d').bz2 |
+curl https://www.balab.aueb.gr/~dds/oasa-2021-01-08.bz2 |
+  bzip2 -d > in.csv

evaluation/log_parsing/nginx_scripts/setup.sh evaluation/benchmarks/analysis-logs/setup.sh

@@ -0,0 +1 @@
+*.txt

evaluation/log_parsing/pcap_scripts/count_packets.sh evaluation/benchmarks/analysis-pcap/count_packets.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+# Auxiliary functions for bi-grams
+
+bigrams_aux()
+{
+    s2=$(mktemp -u)
+    mkfifo $s2
+    tee $s2 |
+        tail -n +2 |
+        paste $s2 - |
+        sed '$d'
+    rm $s2
+}
+
+bigram_aux_map()
+{
+    IN=$1
+    OUT=$2
+    AUX_HEAD=$3
+    AUX_TAIL=$4
+
+    s2=$(mktemp -u)
+    aux1=$(mktemp -u)
+    aux2=$(mktemp -u)
+    aux3=$(mktemp -u)
+    temp=$(mktemp -u)
+
+    mkfifo $s2
+    mkfifo $aux1
+    mkfifo $aux2
+    mkfifo $aux3
+
+    ## New way of doing it using an intermediate file. This is slow
+    ## but doesn't deadlock
+    cat $IN > $temp
+
+    sed '$d' $temp > $aux3 &
+    cat $temp | head -n 1 > $AUX_HEAD &
+    cat $temp | tail -n 1 > $AUX_TAIL &
+    cat $temp | tail -n +2 | paste $aux3 - > $OUT &
+
+    # ## Old way of doing it
+    # cat $IN |
+    #     tee $s2 $aux1 $aux2 |
+    #     tail -n +2 |
+    #     paste $s2 - > $OUT &
+
+    # ## The goal of this is to write the first line of $IN in the $AUX_HEAD
+    # ## stream and the last line of $IN in $AUX_TAIL
+
+    # cat $aux1 | ( head -n 1 > $AUX_HEAD; $PASH_TOP/evaluation/tools/drain_stream.sh ) &
+    # # while IFS= read -r line
+    # # do
+    # #     old_line=$line
+    # # done < $aux2
+    # # echo "$old_line" > $AUX_TAIL
+    # ( tail -n 1 $aux2 > $AUX_TAIL; $PASH_TOP/evaluation/tools/drain_stream.sh ) &
+
+    wait
+
+    rm $temp
+    rm $s2
+    rm $aux1
+    rm $aux2
+    rm $aux3
+}
+
+bigram_aux_reduce()
+{
+    IN1=$1
+    AUX_HEAD1=$2
+    AUX_TAIL1=$3
+    IN2=$4
+    AUX_HEAD2=$5
+    AUX_TAIL2=$6
+    OUT=$7
+    AUX_HEAD_OUT=$8
+    AUX_TAIL_OUT=$9
+
+    temp=$(mktemp -u)
+
+    mkfifo $temp
+
+    cat $AUX_HEAD1 > $AUX_HEAD_OUT &
+    cat $AUX_TAIL2 > $AUX_TAIL_OUT &
+    paste $AUX_TAIL1 $AUX_HEAD2 > $temp &
+    cat $IN1 $temp $IN2 > $OUT &
+
+    wait
+
+    rm $temp
+}
+
+export -f bigrams_aux
+export -f bigram_aux_map
+export -f bigram_aux_reduce

evaluation/log_parsing/pcap_scripts/pcap_bench.sh evaluation/benchmarks/analysis-pcap/pcap_bench.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Find all 2-grams in a piece of text
+
+IN=${IN:-$PASH_TOP/evaluation/benchmarks/expert-oneliners/10G.txt}
+
+. bi-gram.aux.sh
+
+cat $IN |
+  tr -cs A-Za-z '\n' |
+  tr A-Z a-z |
+  bigrams_aux |
+  sort |
+  uniq
+
+

evaluation/log_parsing/pcap_scripts/setup.sh evaluation/benchmarks/analysis-pcap/setup.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Compares two streams element by element
+# Taken from https://crashingdaily.wordpress.com/2008/03/06/diff-two-stdout-streams/
+# shuf() { awk 'BEGIN {srand(); OFMT="%.17f"} {print rand(), $0}' "$@" | sort -k1,1n | cut -d ' ' -f2-; }
+
+IN=${IN:-$PASH_TOP/evaluation/benchmarks/expert-oneliners/10G.txt}
+
+mkfifo s1 s2
+
+cat $IN |
+  # shuf |
+  tr [:lower:] [:upper:] |
+  sort > s1 &
+
+cat $IN |
+  # shuf |
+  tr [:upper:] [:lower:] |
+  sort > s2 &
+
+diff -B s1 s2
+rm s1 s2

evaluation/log_parsing/pcap_scripts/split_pcap.sh evaluation/benchmarks/analysis-pcap/split_pcap.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# Match complex regular-expression over input
+
+IN=${IN:-$PASH_TOP/evaluation/benchmarks/expert-oneliners/10G.txt}
+
+cat $IN | tr A-Z a-z | grep '\(.\).*\1\(.\).*\2\(.\).*\3\(.\).*\4'

evaluation/buses/.gitignore evaluation/benchmarks/analytics-mts/.gitignore

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Show the set-difference between two streams (i.e., elements in the first that are not in the second).
+# https://stackoverflow.com/questions/2509533/bash-linux-set-difference-between-two-text-files
+
+IN=${IN:-$PASH_TOP/evaluation/benchmarks/expert-oneliners/10G.txt}
+
+mkfifo s1 s2
+
+cat $IN |
+    cut -d ' ' -f 1 |
+    tr [:lower] [:upper] |
+    sort > s1 &
+
+cat $IN |
+    cut -d ' ' -f 1 |
+    sort > s2 &
+
+comm -23 s1 s2
+
+rm s1 s2

evaluation/buses/1.sh evaluation/benchmarks/analytics-mts/1.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -e
+
+# call the script with its absolute name
+cd $(dirname $0)
+
+curl 'http://ndr.md/data/dummy/1M.txt' > 1M.txt
+# download wamerican-insane dictionary and sort according to machine
+curl 'http://ndr.md/data/dummy/dict.txt' | sort > dict.txt
+
+touch 10M.txt
+for (( i = 0; i < 10; i++ )); do
+  cat 1M.txt >> 10M.txt
+done
+
+touch 100M.txt
+for (( i = 0; i < 10; i++ )); do
+  cat 10M.txt >> 100M.txt
+done
+
+touch 1G.txt
+for (( i = 0; i < 10; i++ )); do
+  cat 100M.txt >> 1G.txt
+done
+
+if [ "$#" -eq 1 ] && [ "$1" = "--full" ]; then
+  echo Generting full-size inputs
+
+  touch 3G.txt
+  for (( i = 0; i < 3; i++ )); do
+    cat 1G.txt >> 3G.txt
+  done
+
+  touch 10G.txt
+  for (( i = 0; i < 10; i++ )); do
+    cat 1G.txt >> 10G.txt
+  done
+
+  touch 100G.txt
+  for (( i = 0; i < 10; i++ )); do
+    cat 10G.txt >> 100G.txt
+  done
+fi

evaluation/buses/2.sh evaluation/benchmarks/analytics-mts/2.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Find the shortest scripts 
+# From "Wicked Cool Shell Scripts", 2nd Ed., pg. 7
+# +p.95 multiple sed
+# +p.XX crawler
+
+IN=${IN:-$PASH_TOP/evaluation/benchmarks/expert-oneliners/10G.txt}
+
+cat $IN | xargs file | grep "shell script" | cut -d: -f1 | xargs -L 1 wc -l | grep -v '^0$' | sort -n | head -15

evaluation/buses/3.sh evaluation/benchmarks/analytics-mts/3.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# Calculate sort twice
+
+IN=${IN:-$PASH_TOP/evaluation/benchmarks/expert-oneliners/10G.txt}
+
+cat $IN | tr A-Z a-z | sort | sort -r

evaluation/buses/4.sh evaluation/benchmarks/analytics-mts/4.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Sort input
+
+IN=${IN:-$PASH_TOP/evaluation/benchmarks/expert-oneliners/10G.txt}
+
+cat $IN | sort
+

evaluation/buses/5.sh evaluation/benchmarks/analytics-mts/5.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Calculate mispelled words in an input
+# https://dl.acm.org/doi/10.1145/3532.315102
+
+IN=${IN:-$PASH_TOP/evaluation/benchmarks/expert-oneliners/10G.txt}
+
+cat $IN |
+    # groff -t -e -mandoc -Tascii |  # remove formatting commands
+    col -bx |                      # remove backspaces / linefeeds
+    tr -cs A-Za-z '\n' |
+    tr A-Z a-z |                   # map upper to lower case
+    tr -d '[:punct:]' |            # remove punctuation
+    sort |                         # put words in alphabetical order
+    uniq |                         # remove duplicate words
+    comm -23 - $dict               # report words not in dictionary 

evaluation/benchmarks/analytics-mts/README.md

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Top-N (1000) terms
+# from https://dl.acm.org/doi/10.1145/5948.315654
+
+IN=${IN:-$PASH_TOP/evaluation/benchmarks/expert-oneliners/10G.txt}
+
+cat $IN | tr -cs A-Za-z '\n' | tr A-Z a-z | sort | uniq -c | sort -rn | sed 100q
+

evaluation/benchmarks/analytics-mts/setup

@@ -0,0 +1,6 @@
+#!/bin/bash
+# Calculate the frequency of each word in the document, and sort by frequency
+
+IN=${IN:-$PASH_TOP/evaluation/benchmarks/expert-oneliners/10G.txt}
+
+cat $IN | tr -cs A-Za-z'\n' | tr A-Z a-z | sort | uniq -c | sort -rn 

evaluation/scripts/bio-align/bio.sh evaluation/benchmarks/bio-align/bio.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
+#Calculate maximum temperature across the US over five years
 
 #NOTE: The `head -n 1 below is for minimizing the number of pages to be seen
 
 # `seq` is similar to {1995..2005}, but this requires shell expansion rules that
 # are quite convoluted
-seq 2005 2005 | 
+seq 2015 2019 | 
   sed 's;^;http://ndr.md/data/noaa/;' |
   sed 's;$;/;' |
   xargs -n 1 curl -s |

evaluation/scripts/genome-diff.sh evaluation/benchmarks/bio-align/genome-diff.sh

@@ -0,0 +1,2 @@
+exodus
+genesis

evaluation/scripts/genquality.sh evaluation/benchmarks/bio-align/genquality.sh

@@ -1,2 +1,3 @@
 inputs/*
 intermediary/*
+*.txt