diff --git a/.github/workflows/bandit-check.yml b/.github/workflows/bandit-check.yml new file mode 100644 index 000000000..e9d7c8379 --- /dev/null +++ b/.github/workflows/bandit-check.yml @@ -0,0 +1,33 @@ +name: Bandit Check +on: + workflow_dispatch: + schedule: + - cron: '5 23 * * 1' + push: + paths: + - .github/workflows/bandit-check.yml + +permissions: read-all + +jobs: + bandit-check: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Python + uses: actions/setup-python@v5 + + - name: Bandit check + run: | + pip install bandit + bandit -c bandit.yml -r ./scripts ./test --exit-zero -f custom --msg-template \ + "{relpath}:{line:<4} {test_id}[bandit]:{severity}: {msg}" + bandit -c bandit.yml -r ./scripts ./test --exit-zero -f html -o bandit_report.html + + - name: Upload report to artifacts + uses: actions/upload-artifact@v4 + with: + name: Bandit Report + path: bandit_report.html diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml new file mode 100644 index 000000000..0a82bebbf --- /dev/null +++ b/.github/workflows/coverity.yml @@ -0,0 +1,164 @@ +name: Coverity + +on: + workflow_dispatch: + schedule: + - cron: "5 2 * * 1" + push: + paths: + - .github/workflows/coverity.yml + +permissions: read-all + +jobs: + coverity: + name: Coverity + runs-on: ubuntu-latest + env: + COV_TOKEN: jQxsG_hSGU57M7A7lqyBvg + LLVM_SHA_FILE: /home/runner/work/mlir-extensions/mlir-extensions/build_tools/llvm_version.txt + + defaults: + run: + shell: bash -leo pipefail {0} + + timeout-minutes: 450 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Cache coverity + id: cache-coverity + uses: actions/cache@v4 + env: + COVERITY_CACHE_NUMBER: 1 # Increase to reset cache + with: + path: | + /home/runner/coverity + key: ${{ runner.os }}-${{ env.COVERITY_CACHE_NUMBER }} + + - name: Setup conda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: ${{ matrix.python }} + activate-environment: imex-devel + + - name: Conda info + run: conda info + + - name: Install Build tools + run: | + conda install cmake ninja conda-forge::lit conda-forge::doxygen + conda list + + - name: Download coverity + if: steps.cache-coverity.outputs.cache-hit != 'true' + run: | + cd + curl -X POST https://scan.coverity.com/download/linux64 -d "token=$COV_TOKEN" --data-urlencode "project=MLIR Extensions" -o coverity.tgz + tar zxf coverity.tgz + mv -T cov-analysis-linux64-* coverity + + - name: Add coverity to PATH + run: | + echo "$HOME/coverity/bin" >> $GITHUB_PATH + + - name: Show coverity version + run: | + coverity --version + + - name: Add coverity inputs to env + run: | + version="$(git rev-parse --short HEAD)" + email="alexei.fedotov@intel.com" + project_id=30755 + + echo "project_id=$project_id" | tee -a $GITHUB_ENV + echo "email=$email" | tee -a $GITHUB_ENV + echo "version=$version" | tee -a $GITHUB_ENV + + - name: Setup Cache Vars + run: | + echo 'LLVM_SHA<> $GITHUB_ENV + cat $LLVM_SHA_FILE >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + + - name: Cache LLLVM-MLIR + id: cache-llvm-mlir + uses: actions/cache@v4 + env: + LLVM_CACHE_NUMBER: 1 # Increase to reset cache + with: + path: | + /home/runner/work/llvm-mlir/_mlir_install/** + key: ${{ runner.os }}-build-llvm-${{ env.LLVM_CACHE_NUMBER }}-${{ env.LLVM_SHA }} + + - name: Build LLVM-MLIR + if: steps.cache-llvm-mlir.outputs.cache-hit != 'true' + run: | + mkdir -p /home/runner/work/llvm-mlir + cd /home/runner/work/llvm-mlir + echo "INFO: Need to rebuild LLVM-MLIR. Previous installation for MLIR not found" + np=`nproc` + echo "INFO: nproc $np" + git clone https://github.com/llvm/llvm-project --branch main --single-branch + cd llvm-project + git checkout ${LLVM_SHA} + git apply /home/runner/work/mlir-extensions/mlir-extensions/build_tools/patches/*.patch + cmake -G Ninja -B _build -S llvm \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_USE_LINKER=gold \ + -DLLVM_INSTALL_UTILS=ON \ + -DLLVM_TARGETS_TO_BUILD=X86 \ + -DLLVM_ENABLE_BINDINGS=OFF \ + -DLLVM_ENABLE_ZSTD=OFF \ + -DCMAKE_INSTALL_PREFIX=/home/runner/work/llvm-mlir/_mlir_install + cmake --build _build --target install + + - name: Create coverity build + run: | + external_lit=`which lit` + echo ${external_lit} + cd /home/runner/work/mlir-extensions/mlir-extensions + cmake -S . -B _build -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_EXTERNAL_LIT=${external_lit} \ + -DMLIR_DIR=/home/runner/work/llvm-mlir/_mlir_install/lib/cmake/mlir \ + -DLLVM_LIT_ARGS=-a + cov-build --dir $HOME/cov-int cmake --build _build --target check-imex + + - name: Create coverity results tarball + run: | + cd $HOME + tar zcf cov-int.tgz cov-int + + - name: Create coverity artifact + run: | + cd $HOME + curl -X POST \ + -d version="$version" \ + -d email="$email" \ + -d token=$COV_TOKEN \ + -d file_name="cov-int.tgz" \ + https://scan.coverity.com/projects/$project_id/builds/init \ + | tee response + upload_url="$(jq -r '.url' response)" + build_id="$(jq -r '.build_id' response)" + echo "upload_url=$upload_url" >> $GITHUB_ENV + echo "build_id=$build_id" | tee -a $GITHUB_ENV + + - name: Upload coverity build + run: | + cd $HOME + curl -X PUT \ + --header 'Content-Type: application/json' \ + --upload-file cov-int.tgz \ + $upload_url + + curl -X PUT \ + -d token=$COV_TOKEN \ + https://scan.coverity.com/projects/$project_id/builds/$build_id/enqueue diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 190740595..4ffeca759 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -11,6 +11,6 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.0 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: pre-commit/action@v3.0.1 diff --git a/bandit.yml b/bandit.yml new file mode 100644 index 000000000..cf5f29ebc --- /dev/null +++ b/bandit.yml @@ -0,0 +1,186 @@ + +### Bandit config file generated from: +# '/bandit-config-generator -o bandit.yaml' + +### This config may optionally select a subset of tests to run or skip by +### filling out the 'tests' and 'skips' lists given below. If no tests are +### specified for inclusion then it is assumed all tests are desired. The skips +### set will remove specific tests from the include set. This can be controlled +### using the -t/-s CLI options. Note that the same test ID should not appear +### in both 'tests' and 'skips', this would be nonsensical and is detected by +### Bandit at runtime. + +# Available tests: +# B101 : assert_used +# B102 : exec_used +# B103 : set_bad_file_permissions +# B104 : hardcoded_bind_all_interfaces +# B105 : hardcoded_password_string +# B106 : hardcoded_password_funcarg +# B107 : hardcoded_password_default +# B108 : hardcoded_tmp_directory +# B110 : try_except_pass +# B112 : try_except_continue +# B113 : request_without_timeout +# B201 : flask_debug_true +# B202 : tarfile_unsafe_members +# B301 : pickle +# B302 : marshal +# B303 : md5 +# B304 : ciphers +# B305 : cipher_modes +# B306 : mktemp_q +# B307 : eval +# B308 : mark_safe +# B310 : urllib_urlopen +# B311 : random +# B312 : telnetlib +# B313 : xml_bad_cElementTree +# B314 : xml_bad_ElementTree +# B315 : xml_bad_expatreader +# B316 : xml_bad_expatbuilder +# B317 : xml_bad_sax +# B318 : xml_bad_minidom +# B319 : xml_bad_pulldom +# B320 : xml_bad_etree +# B321 : ftplib +# B323 : unverified_context +# B324 : hashlib_insecure_functions +# B401 : import_telnetlib +# B402 : import_ftplib +# B403 : import_pickle +# B404 : import_subprocess +# B405 : import_xml_etree +# B406 : import_xml_sax +# B407 : import_xml_expat +# B408 : import_xml_minidom +# B409 : import_xml_pulldom +# B410 : import_lxml +# B411 : import_xmlrpclib +# B412 : import_httpoxy +# B413 : import_pycrypto +# B415 : import_pyghmi +# B501 : request_with_no_cert_validation +# B502 : ssl_with_bad_version +# B503 : ssl_with_bad_defaults +# B504 : ssl_with_no_version +# B505 : weak_cryptographic_key +# B506 : yaml_load +# B507 : ssh_no_host_key_verification +# B508 : snmp_insecure_version +# B509 : snmp_weak_cryptography +# B601 : paramiko_calls +# B602 : subprocess_popen_with_shell_equals_true +# B603 : subprocess_without_shell_equals_true +# B604 : any_other_function_with_shell_equals_true +# B605 : start_process_with_a_shell +# B606 : start_process_with_no_shell +# B607 : start_process_with_partial_path +# B608 : hardcoded_sql_expressions +# B609 : linux_commands_wildcard_injection +# B610 : django_extra_used +# B611 : django_rawsql_used +# B612 : logging_config_insecure_listen +# B701 : jinja2_autoescape_false +# B702 : use_of_mako_templates +# B703 : django_mark_safe + +# (optional) list included test IDs here, eg '[B101, B406]': +tests: [] + +# (optional) list skipped test IDs here, eg '[B101, B406]': +skips: [] + +### (optional) plugin settings - some test plugins require configuration data +### that may be given here, per-plugin. All bandit test plugins have a built in +### set of sensible defaults and these will be used if no configuration is +### provided. It is not necessary to provide settings for every (or any) plugin +### if the defaults are acceptable. +assert_used: + skips: ['./benchmarks/*'] # accept those assert in test scripts +hardcoded_tmp_directory: + tmp_dirs: + - /tmp + - /var/tmp + - /dev/shm +# subprocess_popen_with_shell_equals_true: #B602 +# subprocess_without_shell_equals_true: #B603 +# any_other_function_with_shell_equals_true: #B604 +# start_process_with_a_shell: #B605 +# start_process_with_no_shell: #B606 +# start_process_with_partial_path: #B607 +# linux_commands_wildcard_injection: #B609 +# test ID B6* shares a configuration in the same family, namely shell_injection +shell_injection: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + - subprocess.getoutput + - subprocess.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +ssl_with_bad_defaults: + bad_protocol_versions: + - PROTOCOL_SSLv2 + - SSLv2_METHOD + - SSLv23_METHOD + - PROTOCOL_SSLv3 + - PROTOCOL_TLSv1 + - SSLv3_METHOD + - TLSv1_METHOD + - PROTOCOL_TLSv1_1 + - TLSv1_1_METHOD +ssl_with_bad_version: + bad_protocol_versions: + - PROTOCOL_SSLv2 + - SSLv2_METHOD + - SSLv23_METHOD + - PROTOCOL_SSLv3 + - PROTOCOL_TLSv1 + - SSLv3_METHOD + - TLSv1_METHOD + - PROTOCOL_TLSv1_1 + - TLSv1_1_METHOD +try_except_continue: + check_typed_exception: false +try_except_pass: + check_typed_exception: false +weak_cryptographic_key: + weak_key_size_dsa_high: 1024 + weak_key_size_dsa_medium: 2048 + weak_key_size_ec_high: 160 + weak_key_size_ec_medium: 224 + weak_key_size_rsa_high: 1024 + weak_key_size_rsa_medium: 2048