msa_build.py

"""
msa_build.py seq.fasta             \\
    --hhblitsdb=uniclust30_2017_10 \\
    --jackhmmerdb=uniref90.fasta   \\
    --bfddb=bfd.fasta              \\
    --hmmsearchdb=metaclust.fasta

    build PSICOV format MSA using hhblits, jackhmmer, or hmmsearch.

options:
    (at least one of -hhblitsdb, --jackhmmerdb or --hmmsearchdb must be set)

    --hhblitsdb=uniclust30_2017_10
        hhsuite databases, to be searched by hhblits.

    --jackhmmerdb=uniref90.fasta
        (decompressed) fasta databases, to be searched by jackhmmer, whose
        search result will be built into a custom hhsuite database to be
        searched by hhblits, jump-starting from alignment generated by
        searching hhblitsdb. fasta database must have an ssi index file
        created by esl-sfetch.

    --hmmsearchdb=metaclust.fasta:tara.fasta
        colon deliminated list of decompressed fasta database, to be
        searched hmmsearch, jump-starting from alignment generated by
        searching either -hhblitsdb or -jackhmmerdb. fasta database must
        have an ssi index file created by esl-sfetch.

    --tmpdir=/tmp/$USER/MSA_`date +%N`
        temporary folder

    --outdir=.
        output folder. default is current folder

    --overwrite={0,1,2,4}
        whether overwrite existing search result.
        0 - do not overwrite any intermediate alignment
        1 - overwrite hhblitsdb search result (.hhbaln and .hhba3m)
        2 - overwrite jackhmmerdb search result (.jacaln and .jaca3m)
        4 - overwrite jackhmmerdb search result (.bfdaln and .bfda3m)
        8 - overwrite hmmsearchdb search result (.hmsaln)
        These options are addictive. For example, -overwrite=7 (=1+2+4) for
        overwriting any intermediate alignment (but might still filter
        final alignment if it is too large).

    --ncpu=1
        number of CPU threads. do not use multi-threading by default.

output:
    (filename prefix determined by input filename)
    seq.aln     - final alignment. the only non-optional output
    seq.hhbaln  - (if --hhblitsdb is set) hhblits MSA (PSICOV format)
    seq.hhba3m  - (if --hhblitsdb is set) hhblits MSA (a3m format)
    seq.jacaln  - (if --jackhmmerdb is set and --hhblitsdb search does not have
                  enough sequences) jackhmmer + hhblits MSA (PSICOV format)
    seq.jaca3m  - (if --jackhmmerdb is set and --hhblitsdb search does not have
                  enough sequences) jackhmmer + hhblits MSA (a3m format)
    seq.bfdaln  - (if --bfddb is set and neither --hhblitsdb nor --jackhmmerdb
                  search has enough sequences)+ hhblits MSA (PSICOV format)
    seq.bfda3m  - (if --bfddb is set and neither --hhblitsdb nor --jackhmmerdb
                  search has enough sequences) hhblits MSA (a3m format)
    seq.hmsaln  - (if --hmmsearchdb is set and neither --hhblitsdb nor
                  --jackhmmerdb search has enough sequences)
                  hhblits + jackhmmer (optional) + hmmsearch output
"""

import os
import sys
import functools
import json
import math
import multiprocessing as mp
import shutil
from string import Template, ascii_lowercase
import subprocess
import logging

from hhpaths import bin_dict
from kclust2db import id2s_dict, kclust2db, remove_a3m_gap, write_fseqs
from utils import exists, make_tmpdir, mkdir_if_not_exist

logger = logging.getLogger(__file__)

#### search parameters ####
# min query coverage by template
cov_cut = [
    50,  # from metapsicov 2.0.3 and deepcontact. for final alignment output
    60,  # from metapsicov 1.04. for Nf calculation
    75,  # from gremlin. for qhmmbuild searching
]

# target nf, beyond which we do not attempt to build deeper alignment
target_nf = [
    128,
    129,
]

# max seqID within MSA
id_cut = [
    99,  # from metapsicov and deepcontact
    90,  # from gremlin
]

# which file is used by qhmmbuild to build query hmm
# 1  - (recommended) use PSICOV format .aln alignment
# 0  - use hhsuite format .a3m alignment, takes more resources with
#      slightly worse result
aln2hmm = 1

# whether to build custom hhsuite database from qhmmsearch hits
# 0 - directly use filtered and realigned qhmmsearch MSA as output
# 1 - build custom hhsuite database from qhmmsearch hits, and search
#     this custom database with hhblits, similar to jackblits
# 2 - build custom hhsuite database from qhmmsearch hits as well as
#     hhblits/jackblits hits, and search this custom database with hhblits
build_hmmsearch_db = 1

# whether to further filter qhmmsearch+hhblits result using more stringent
# id_cut and cov_cut
# 0 - no further filter
# 1 - filter if nf>=target_nf
# 2 - always filter
filter_hmsblits = 0

# The threshold above which kClust'ed hhsuite database is build instead of
# non-clustered hhsuite database. For some reason, non-clustered database
# has slightly better alignment quality, but takes much longer time to make.
kclust2db_threshold = 1000

# The maximum number of hits to be parsed in a checkpoint alignment
checkali_threshold = 30000

#### command templates ####
# Although both gremlin and PconSC2 recommend increasing -maxfilt, it seems
# higher -maxfilt increases alignment depth without significant benefit for
# improving contact prediction accuracy. Therefore, -maxfilt is still
# default value here.
# $infile      - input fasta or a3m
# $db          - hhsuite database
# $ncpu        - number of cpu cores
# $outprefix   - outputs are $outprefix.a3m $outprefix.log $outprefix.aln
# $id_cut      - percentage max seqID among aligned sequences
# $cov_cut     - percentage min cov of query by templates
hhblits_template = Template(bin_dict["hhblits"] + \
    " -i $infile -diff inf -d $db -cpu $ncpu -oa3m $outprefix.a3m -e $e" + \
    " -id $id_cut -cov $cov_cut -o $outprefix.log -n 3 -diff inf; "      + \
    "grep -v '^>' $outprefix.a3m|sed 's/[a-z]//g' > $outprefix.aln")

# While both hhfilter and rmRedundantSeq can control the max seqID and min
# coverage within an alignment, they are slightly different: hhfilter
# consider seqID normalized by full query length, while rmRedundantSeq
# consider seqID normalized by aligned region only.
# $prefix      - input in $prefix.a3m (hhfilter_template) or
#                in $prefix.aln (alnfilter_template)
#                output are $prefix.$cov_cut.a3m and $prefix.$cov_cut.aln
# $id_cut      - percentage max seqID among aligned sequences
# $cov_cut     - percantage min cov of query by templates
hhfilter_template = Template(bin_dict["hhfilter"]                           + \
    " -i $prefix.a3m -o $prefix.$cov_cut.a3m -id $id_cut -cov $cov_cut; "   + \
    "grep -v '^>' $prefix.$cov_cut.a3m|sed 's/[a-z]//g' > $prefix.$cov_cut.aln")

alnfilter_template = Template(bin_dict["rmRedundantSeq"] + \
    " $id_cut $cov_cut $prefix.aln > $prefix.$cov_cut.aln")

# $calNf       - calNf executable
# $infile      - PSICOV format input
# $target_nf   - max Nf to consider
calNf_template = Template(bin_dict["calNf"] + " $infile 0.8 0 $target_nf")

# qjackhmmer is almost identical to jackhmmer, except that the output
# alignment only include positions covered by query. Here, the number of
# sequences in ".tbl" and ".fseqs" files is usually larger than in ".first"
# file, because tbl and fseqs contain all hits with -E <= 10, while
# ".first" only contain --incE <= 1e-3. Short queries might hit a long-
# multidomain template with significant -E but not significant --incE. This
# neccesitate trim_eslsfetch for removing spurious fseqs hits and template
# regions that are neither aligned nor close to an aligned region.
# $ncpu        - number of cpu cores
# $outprefix   - outputs are $outprefix.first $outprefix.tbl
#                $outprefix.fseqs $outprefix.out
# $infile      - input query fasta
# $db          - jackhmmer database
qjackhmmer_template = Template(bin_dict["qjackhmmer"]+" --cpu $ncpu"      + \
    " -N 3 -E 10 --incE 1e-3 -A $outprefix.first --tblout $outprefix.tbl" + \
    " -o $outprefix.out $infile $db; "+bin_dict["eslsfetch"]              + \
    " -f $db $outprefix.tbl|sed 's/*//g' > $outprefix.fseqs")

# hhblitsdb.pl can take much longer time than even jackhmmer if there
# are many hits. In this case, the hits are first clustered by kClust
# before being used by hhblitsdb.pl.
# $ncpu        - number of cpu cores
# $db          - output hhblits database
# $a3mdir      - input a3m folder
hhblitsdb_template = Template(bin_dict["hhblitsdb"] +
                              " --cpu $ncpu -o $db --input_a3m $a3mdir")

# qhmmbuild is almost identical to hmmbuild, except that the output
# alignment only contains the first sequence in query alignment, so
# as to save disk space. qhmmbuild can be replaced by hmmbuild.
# $infile      - input a3m (qhmmbuild_a3m_template) or
#                input aln (qhmmbuild_aln_template)
# $outprefix   - outputs are $outprefix.afq $outprefix.hmm
qhmmbuild_aln_template = Template(
    "sed = $infile |sed 'N;s/\\n/\\t/'|sed 's/^/>/g'|sed 's/\\t/\\n/g'|"+ \
    bin_dict["qhmmbuild"]                                               + \
    " -n aln --amino -O $outprefix.afq --informat afa $outprefix.hmm -")

qhmmbuild_a3m_template = Template(
    bin_dict["reformat"]+" a3m a2m $infile -|grep -vP "                 + \
    "'^(Reformat|Using |Removed |Sequence |inserting |WARNING: )'|"     + \
    bin_dict["qhmmbuild"]                                               + \
    " -n a3m --amino -O $outprefix.afq --informat afa $outprefix.hmm -")

# This only check number of match states that are in query. Match states
# corresponding to gap, i.e. [-], in query are not considered.
# $infile      - input fasta alignment
num_match_state_template = Template(bin_dict["fasta2aln"] +
                                    " $infile|head -1|grep -ohP '[A-Z]'|wc -l")

# qhmmsearch is almost identical to hmmsearch, except that the output
# alignment exclude insertions, which saves lots of disk space.
# $outprefix   - outputs are $outfile.match $outfile.tbl $outprefix.out
# $infile      - input hmm
# $db          - hmmsearch database
# $ncpu        - number of cpu cores
qhmmsearch_template = Template(bin_dict["qhmmsearch"]+" --cpu $ncpu"      + \
    " -A $outprefix.match --incT 27 -T 27 --incdomT 27 -o $outprefix.out" + \
    " --tblout $outprefix.tbl $infile $db")

qhmmsearch_eslsfetch_template = Template(bin_dict["qhmmsearch"]+" --cpu"   + \
    " $ncpu -E 10 --incE 1e-3 -A $outprefix.match --tblout $outprefix.tbl" + \
    " -o $outprefix.out $infile $db; "+bin_dict["eslsfetch"]               + \
    " -f $db $outprefix.tbl|sed 's/*//g' > $outprefix.fseqs")

# The HMM built by qhmmbuild is usually shorter than the query. This is
# because some positions are considered match states while other positions
# are considered insertion states. This result in the qhmmsearch alignment
# being shorter than the query. realignMSA program re-align the qhmmsearch
# alignment to query so that the former has the same length as query.
# $input_match - input fasta alignment
# $input_afq   - input match states for query
# $outfile     - output alignment
# $cov_cut1    - coverage of match state positions
# $cov_cut2    - coverage of full query. $cov_cut2<=$cov_cut1
realignMSA_template = Template("cat $input_match|sed 's/[*JUZBOX]/-/g'|" + \
    bin_dict["fastaCov"]+" $cov_cut1|"                                   + \
    bin_dict["realignMSA"]+" $input_afq - |"                             + \
    bin_dict["fastaCov"]+" $cov_cut2|"                                   + \
    bin_dict["fasta2aln"]+" -|tr '[:lower:]' '[:upper:]' > $outfile")

# $id_cut      - max seqID at aligned region
# $cov_cut     - min cov to query by new template
# $infile1     - old alignment from hhblits or jachmmer
# $infile2     - new alignment from hmmsearch
# $outprefix   - $outprefix.nonredundant is alignment non-redundant
#                to $infile1 and non-redundant within the alignment.
#                $outprefix.aln combines $infile1 and $outprefix.nr
rmRedundantSeq_template = Template(bin_dict["rmRedundantSeq"]        + \
    " $id_cut $cov_cut $infile1 $infile2 > $outprefix.nonredundant;" + \
    " cat $infile1 $outprefix.nonredundant > $outprefix.aln")

#### check databases ####


def check_db(db_dict):  # pylint: disable=redefined-outer-name
  ''' check if databases are legal '''
  if "hhblitsdb" not in db_dict:
    logger.warning("Not setting --hhblitsdb is unrecommended.")
    if "jackhmmerdb" not in db_dict:
      if "hmmsearchdb" in db_dict:
        logger.warning(
            "Using single query sequence for hmmsearch generates worse result")
      else:
        logger.error("Please at least set --hhblitsdb")
        logger.error("No database to search!")
        sys.exit()

  if "hhblitsdb" in db_dict:
    for db in db_dict["hhblitsdb"].split(","):
      a3m_db = db + "_a3m.ffdata"
      if not os.path.isfile(a3m_db):
        logger.error("Cannot locate %s for --hhblitsdb=%s", a3m_db,
                     db_dict["hhblitsdb"])
        sys.exit()

  if "jackhmmerdb" in db_dict:
    for db in db_dict["jackhmmerdb"]:
      if not os.path.isfile(db):
        logger.error("No such jackhmmerdb file %s", db)
        sys.exit()

      if db.endswith(".gz") or db.endswith(".bz2"):
        logger.error("jackhmmer cannot use compressed file %s", db)
        sys.exit()

      ssi_db = db + ".ssi"
      if not os.path.isfile(ssi_db):
        logger.error("ssi index file missing for %s.", db)
        logger.error("You can create the ssi index file by:")
        logger.error("%s --index %s", bin_dict["eslsfetch"], db)
        sys.exit()

  if "bfddb" in db_dict:
    for db_list in db_dict["bfddb"]:
      for db in db_list.split(","):
        a3m_db = db + "_a3m.ffdata"
        if not os.path.isfile(a3m_db):
          logger.error("Cannot locate %s for --bfddb=%s", a3m_db,
                       db_dict["bfddb"])
          sys.exit()


  if "hmmsearchdb" in db_dict:
    for db in db_dict["hmmsearchdb"]:
      if not os.path.isfile(db):
        logger.error("No such hmmsearchdb file %s", db)
        sys.exit()

      if not build_hmmsearch_db:
        continue

      if db.endswith(".gz") or db.endswith(".bz2"):
        logger.error("esl-sfetch cannot use compressed file %s", db)
        sys.exit()

      ssi_db = db + ".ssi"
      if not os.path.isfile(ssi_db):
        logger.error("ssi index file missing for %s.", db)
        logger.error("You can create the ssi index file by:")
        logger.error("%s --index %s", bin_dict["eslsfetch"], db)
        sys.exit()


#### parse query ####


def read_one_sequence(query_fasta="seq.fasta"):  # pylint: disable=redefined-outer-name
  ''' check if input is legal single sequence fasta and read the sequence '''
  with open(query_fasta, "r") as fp:
    txt = fp.read()
  if ("\n" + txt).count("\n>") != 1:
    logger.error("Input is not single sequence fasta.")
    sys.exit()
  seq = ""
  for line in txt.splitlines():
    if not line.startswith(">"):
      seq += line.strip()
  seq = seq.upper().replace(" ", "").replace("\t", "")
  illegal_residues = set(seq) - set("ABCDEFGHIKLMNOPQRSTUVWXYZ")
  if illegal_residues:
    logger.error("%s contains illegal residues %s", query_fasta,
                 " ".join(illegal_residues))
    sys.exit()
  return seq


#### search sequence database ####


def get_neff(prefix):  # pylint: disable=redefined-outer-name
  ''' return Nf on -cov 60 filtered MSA. input file is prefix.a3m.
    output files are prefix.60.a3m and prefix.60.aln
    '''
  cov = cov_cut[1]
  infile = f"{prefix}.{cov}.aln"

  #### filter input alignment ####
  if not os.path.isfile(infile):
    if os.path.isfile(prefix + ".a3m"):
      cmd = hhfilter_template.substitute(
          prefix=prefix,
          id_cut=id_cut[0],
          cov_cut=cov,
      )
    else:
      cmd = alnfilter_template.substitute(
          prefix=prefix,
          id_cut=id_cut[0],
          cov_cut=cov,
      )
    logger.info(cmd)
    os.system(cmd)

  #### calculate Nf ####
  cmd = calNf_template.substitute(
      infile=infile,
      target_nf=target_nf[-1],
  )
  with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as p:
    stdout, _ = p.communicate()
  return float(stdout)


def run_command_fn(cmd_template, cmd_prefix, db_list, db_idx, **kwargs):
  db = db_list[db_idx]
  outprefix = cmd_prefix + f".{db_idx}"
  cmd = cmd_template.substitute(db=db, outprefix=outprefix, **kwargs)
  logger.info(cmd)
  os.system(cmd)
  return db_idx, outprefix


def run_command_mp(cmd_template, cmd_prefix, db_list, **kwargs):
  command_fn = functools.partial(run_command_fn, cmd_template, cmd_prefix,
                                 db_list, **kwargs)
  with mp.Pool(processes=len(db_list)) as p:
    yield from p.imap(command_fn, range(len(db_list)))


def hhblitsdb_params(db):
  return " -d ".join(db.split(","))


def run_hhblits(query_fasta, db, ncpu, hhblits_prefix):  # pylint: disable=redefined-outer-name
  ''' run hhblits with -cov 50 and return Nf for -cov 60'''
  # outputs are $outprefix.a3m $outprefix.log $outprefix.aln
  cmd = hhblits_template.substitute(
      infile=query_fasta,
      db=hhblitsdb_params(db),
      ncpu=ncpu,
      outprefix=hhblits_prefix,
      id_cut=id_cut[0],    # 99 in metapsicov
      cov_cut=cov_cut[0],  # 50 in metapsicov 2.0.3
      e=0.001,             # E-value cutoff (def=0.001)
 )
  logger.info(cmd)
  os.system(cmd)
  return get_neff(hhblits_prefix)


def fasta2a3msplit(txt, a3mdir="."):
  ''' split multiple sequence FASTA txt into single sequence a3m files
    under "a3mdir"'''
  mkdir_if_not_exist(a3mdir)
  seqnum = 0
  for block in ("\n" + txt).split("\n>"):
    lines = block.splitlines()
    if len(lines) < 2:
      continue
    name = lines[0].split()[0].replace("|", "_")
    seq = "".join(lines[1:])
    with open(os.path.join(a3mdir, name + ".a3m"), "w") as fp:
      fp.write(">" + lines[0] + "\n" + seq + "\n")
    seqnum += 1
  return seqnum


def trim_eslsfetch(fseqs_file,
                   first_file,
                   L=0,  # pylint: disable=invalid-name
                   seqname_prefix="",
                   max_seqnum=0):
  '''
    trim fseqs_file according to first_file so that for a template in
    fseqs_file, the N and C termini are trimmed such that template
    sequence flanking the aligned region is <=L at each side.

    fseqs_file - fasta fetched by esl-efetch result
    first_file - alignment output by qjackhmmer
    L          - length of query. If L is 0, its value is inferred from the
                 first sequence in first_file. This first sequence will be
                 assumed to be the query. If L is not 0, all sequences in
                 first_file will be assumed to be templates
    seqname_prefix - append this to the begining of each sequence name. This
                 is mainly for avoiding name conflict, i.e. sequences from
                 different databases sharing the same name.
    max_seqnum - maximum number of hits to be parsed in first_file
                 default is 0, which means parsing all sequences
    '''
  trim_txt = ""
  L = 0
  trim_dict = dict()  # key is sequence name, value (min,max) position to keep

  seqnum = 0
  with open(first_file, "r") as fp:
    for block in ("\n" + fp.read()).split("\n>"):
      lines = block.splitlines()
      if len(lines) < 2:
        continue
      if not L:
        seq = "".join(lines[1:])
        L = len(seq)
        trim_dict[lines[0].split()[0]] = (1, L)
      else:
        subseq_name = lines[0].split()[0]
        name, pos_range = subseq_name.split("/")
        min_pos, max_pos = map(int, pos_range.split("-"))
        min_pos = max([0, min_pos - 1 - L])  # starting from 0
        max_pos += L  # starting from 1
        if name in trim_dict:
          min_pos = min([min_pos, trim_dict[name][0]])
          max_pos = max([max_pos, trim_dict[name][1]])
        trim_dict[name] = (min_pos, max_pos)
      seqnum += 1
      if 0 < max_seqnum <= seqnum:
        break

  Lpart = max([10000, L * 2])  # pylint: disable=invalid-name
  with open(fseqs_file, "r") as fp:
    for block in ("\n" + fp.read()).split("\n>"):
      lines = block.splitlines()
      if len(lines) < 2:
        continue
      seq = "".join(lines[1:])
      name = lines[0].split()[0]
      if not name in trim_dict:
        continue
      seq = seq[trim_dict[name][0]:trim_dict[name][1]]
      # hhmake "helpfully" assume a3m with sequences name ending in
      # "_consensus" must have a master sequence. see "hhalignment.C"
      if "_consensus" in name:
        name = name.replace("_consensus", "_conxen___")
      # hhblitsdb.pl cannot handle more than 32763 positions
      # see src/hhalignment.C
      if len(seq) < Lpart:
        trim_txt += f">{seqname_prefix}{name}\n{seq}\n"
        continue
      for i in range(0, L, Lpart // 2):
        start_pos = Lpart * i // 2
        end_pos = start_pos + Lpart
        trim_txt += f">{seqname_prefix}{name}_{i}\n{seq[start_pos:end_pos]}\n"  # pylint: disable=line-too-long
        if end_pos >= len(seq):
          break
  return trim_txt


def run_jackblits(query_fasta, db_list, ncpu, hhblits_prefix, jackblits_prefix):  # pylint: disable=redefined-outer-name
  ''' run jackhmmer, hhblitsdb and hhblits '''
  with open(query_fasta, "r") as fp:
    txt = fp.read()

  #### run jackhmmer ####
  # for d, db in enumerate(db_list):
  #   # outputs are $outprefix.first $outprefix.tbl $outprefix.fseqs
  #   outprefix = jackblits_prefix + f".{d}"
  #   cmd = qjackhmmer_template.substitute(
  #       ncpu=ncpu,
  #       outprefix=outprefix,
  #       infile=query_fasta,
  #       db=db,
  #   )
  #   logger.info(cmd)
  #   os.system(cmd)
  for d, outprefix in run_command_mp(qjackhmmer_template,
                                     jackblits_prefix,
                                     db_list,
                                     ncpu=ncpu,
                                     infile=query_fasta):
    # parse jackhmmer hits
    txt += trim_eslsfetch(
        outprefix + ".fseqs",
        outprefix + ".first",
        seqname_prefix=f"jac{d}_",
        max_seqnum=checkali_threshold,  # avoid excessive number of hits
    )

  #### build hhsuite database ####
  db = f"{jackblits_prefix}-mydb/mydb"

  a3mdir = f"{jackblits_prefix}.fseqs"
  write_fseqs(a3mdir, txt)
  # with open(a3mdir, "w") as fp:
  #   fp.write(txt)

  if txt.count("\n>") > kclust2db_threshold:
    ### cluster at 30% seqID and build database ###
    a3mdir = kclust2db(a3mdir,
                       s=id2s_dict[30],
                       ncpu=ncpu,
                       tmpdir=jackblits_prefix)
  # else:
  #   ### split jackhmmer hits into a3m ###
  #   # a3mdir = jackblits_prefix + "-mya3m"
  #   # fasta2a3msplit(txt, a3mdir)

  ### build single sequence profile database ###
  cmd = hhblitsdb_template.substitute(
      ncpu=ncpu,
      db=db,
      a3mdir=a3mdir,
  )
  logger.info(cmd)
  os.system(cmd)

  #### hhblits search  ####
  if os.path.isfile(hhblits_prefix + ".a3m"):
    query_fasta = hhblits_prefix + ".a3m"
  else:
    logger.warning("Using single sequence for jackblits")
  return run_hhblits(query_fasta, db, ncpu, jackblits_prefix)


def run_bfd(query_fasta, db_list, ncpu, hhblits_prefix, jackblits_prefix,
            bfd_prefix):
  ''' run hhblits searching bfddb'''
  with open(query_fasta, "r") as fp:
    txt = fp.read()

  #### run hhblits ####
  # for d, db in enumerate(db_list):
  #   # outputs are $outprefix.first $outprefix.tbl $outprefix.fseqs
  #   outprefix = bfd_prefix + f".{d}"
  #   cmd = hhblits_template.substitute(
  #       infile=query_fasta,
  #       db=db,
  #       ncpu=ncpu,
  #       outprefix=outprefix, # outputs are $outprefix.a3m
  #       # $outprefix.log $outprefix.aln
  #       id_cut=id_cut[0],    # 99 in metapsicov
  #       cov_cut=cov_cut[0],  # 50 in metapsicov 2.0.3
  #       e=1,                 # E-value cutoff (def=0.001)
  #   )
  #   logger.info(cmd)
  #   os.system(cmd)
  for d, outprefix in run_command_mp(hhblits_template,
                                     bfd_prefix,
                                     [hhblitsdb_params(db) for db in db_list],
                                     ncpu=ncpu,
                                     infile=query_fasta,
                                     id_cut=id_cut[0],
                                     cov_cut=cov_cut[0],
                                     e=1):
    # parse bfd hits
    remove_a3m_gap(outprefix + ".a3m",
                   outprefix + ".fseqs",
                   seqname_prefix=f"bfd{d}_")
    with open(outprefix + ".fseqs", "r") as fp:
      txt += fp.read()

  #### build hhsuite database ####
  db = f"{bfd_prefix}-mydb/mydb"

  a3mdir = f"{bfd_prefix}.fseqs"
  write_fseqs(a3mdir, txt)
  # with open(a3mdir, "w") as fp:
  #   fp.write(txt)

  if txt.count("\n>") > kclust2db_threshold:
    ### cluster at 30% seqID and build database ###
    a3mdir = kclust2db(a3mdir,
                       s=id2s_dict[30],
                       ncpu=ncpu,
                       tmpdir=bfd_prefix)
  # else:
  #   ### split jackhmmer hits into a3m ###
  #   # a3mdir = jackblits_prefix + "-mya3m"
  #   # fasta2a3msplit(txt, a3mdir)

  ### build single sequence profile database ###
  cmd = hhblitsdb_template.substitute(
      ncpu=ncpu,
      db=db,
      a3mdir=a3mdir,
  )
  logger.info(cmd)
  os.system(cmd)

  #### hhblits search  ####
  if os.path.isfile(jackblits_prefix + ".a3m"):
    query_fasta = jackblits_prefix + ".a3m"
  elif os.path.isfile(hhblits_prefix + ".a3m"):
    query_fasta = hhblits_prefix + ".a3m"
  else:
    logger.warning("Using single sequence for bfd hhblits search")
  return run_hhblits(query_fasta, db, ncpu, bfd_prefix)


def run_hmmsearch(
    query_fasta, sequence, hhblits_prefix, db_list, ncpu,  # pylint: disable=redefined-outer-name
    hmmsearch_prefix):  # build_hmmsearch_db==0 in search_metaclust
  del query_fasta  # not used
  L = len(sequence)  # pylint: disable=invalid-name

  #### check number of matched states in query #####
  cmd = num_match_state_template.substitute(infile=hmmsearch_prefix + ".afq")
  logger.info(cmd)
  with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as p:
    stdout, _ = p.communicate()
  Lmatch = int(stdout)  # pylint: disable=invalid-name
  target_Lmatch = 1. * cov_cut[-2] / cov_cut[-1]  # pylint: disable=invalid-name

  #### re-generate hmm if too few match states ####
  if Lmatch < (target_Lmatch * L):
    logger.warning("HMM length %.2f L < %.2f L", 1. * Lmatch / L, target_Lmatch)
    cov = cov_cut[-1]
    if aln2hmm:
      cmd = ";".join([
          alnfilter_template.substitute(
              prefix=hhblits_prefix,
              cov_cut=cov,
              id_cut=100,
          ),
          qhmmbuild_aln_template.substitute(
              infile=hhblits_prefix + f".{cov}.aln",
              outprefix=hmmsearch_prefix,
          ),
      ])
    else:
      cmd = ";".join([
          hhfilter_template.substitute(
              prefix=hhblits_prefix,
              cov_cut=cov,
              id_cut=100,
          ),
          qhmmbuild_a3m_template.substitute(
              infile=hhblits_prefix + f".{cov}.a3m",
              outprefix=hmmsearch_prefix,
          ),
      ])
    logger.info(cmd)
    os.system(cmd)

  #### search hmm against metaclust ####
  # for d, db in enumerate(db_list):
  #   outprefix = hmmsearch_prefix + f".{d}"
  #   cmd = qhmmsearch_template.substitute(
  #       outprefix=outprefix,
  #       infile=hmmsearch_prefix + ".hmm",
  #       ncpu=ncpu,
  #       db=db,
  #   )
  #   logger.info(cmd)
  #   os.system(cmd)
  for d, outprefix in run_command_mp(qhmmsearch_template,
                          hmmsearch_prefix,
                          db_list,
                          ncpu=ncpu,
                          infile=hmmsearch_prefix + ".hmm"):
    del d, outprefix  # dummy code here.

  ### cat hmmsearch aln from multiple databases ###
  cmd = "cat " + " ".join([
      f"{hmmsearch_prefix}.{d}.match" for d in range(len(db_list))
  ]) + f" > {hmmsearch_prefix}.match"
  logger.info(cmd)
  os.system(cmd)

  ### realign MSA to query hmm ###
  cmd = realignMSA_template.substitute(
      input_match=hmmsearch_prefix + ".match",
      input_afq=hmmsearch_prefix + ".afq",
      outfile=hmmsearch_prefix + ".redundant",
      cov_cut1=cov_cut[-1],
      cov_cut2=cov_cut[-2],
  )
  logger.info(cmd)
  os.system(cmd)

  ### remove redundancy in resulting msa ###
  cmd = rmRedundantSeq_template.substitute(
      id_cut=id_cut[-1],  # typically the same or more stringent
      # than that in hhblits
      cov_cut=cov_cut[-2],  # less stringenet than cov_cut1
      infile1=hhblits_prefix + ".aln",
      infile2=hmmsearch_prefix + ".redundant",
      outprefix=hmmsearch_prefix,
  )
  logger.info(cmd)
  os.system(cmd)
  return get_neff(hmmsearch_prefix)


def run_hmsblits(query_fasta, sequence, hhblits_prefix, db_list, ncpu,  # pylint: disable=redefined-outer-name
                 hmmsearch_prefix):  # build_hmmsearch_db in [1,2]
  #### read in query profile ####
  if build_hmmsearch_db == 2 and os.path.isfile(hhblits_prefix + ".a3m"):
    txt = ""
    with open(hhblits_prefix + ".a3m", "r") as fp:
      for line in fp.read().splitlines():
        if line.startswith(">"):
          txt += line + "\n"
        else:
          txt += line.replace("-", "").replace(".", "").upper() + "\n"
  else:
    with open(query_fasta, "r") as fp:
      txt = fp.read()

  #### search hmm against metaclust ####
  # for d, db in enumerate(db_list):
  #   outprefix = hmmsearch_prefix + f".{d}"
  #   cmd = qhmmsearch_eslsfetch_template.substitute(
  #       outprefix=outprefix,
  #       infile=hmmsearch_prefix + ".hmm",
  #       ncpu=ncpu,
  #       db=db,
  #   )
  #   logger.info(cmd)
  #   os.system(cmd)
  for d, outprefix in run_command_mp(qhmmsearch_eslsfetch_template,
                                     hmmsearch_prefix,
                                     db_list,
                                     ncpu=ncpu,
                                     infile=hmmsearch_prefix + ".hmm"):
    txt += trim_eslsfetch(
        outprefix + ".fseqs",
        outprefix + ".match",
        L=len(sequence),
        seqname_prefix=f"hms{d}_",
        max_seqnum=checkali_threshold,  # avoid excessive number of hits
    )

  #### build custom hhsuite database ####
  db = hmmsearch_prefix + "-mydb/mydb"
  mkdir_if_not_exist(db)

  a3mdir = f"{hmmsearch_prefix}.fseqs"
  write_fseqs(a3mdir, txt)
  # with open(a3mdir, "w") as fp:
  #   fp.write(txt)

  if txt.count("\n>") > kclust2db_threshold:
    ### cluster at 30% seqID and build database ###
    a3mdir = kclust2db(a3mdir,
                       s=id2s_dict[30],
                       ncpu=ncpu,
                       tmpdir=hmmsearch_prefix)
  # else:
  #   ### split hmmsearch hits into a3m ###
  #   # a3mdir = hmmsearch_prefix + "-mya3m"
  #   # fasta2a3msplit(txt, a3mdir)

  ### build single sequence profile database ###
  cmd = hhblitsdb_template.substitute(
      ncpu=ncpu,
      db=db,
      a3mdir=a3mdir,
  )
  logger.info(cmd)
  os.system(cmd)

  #### hhblits search  ####
  if not os.path.isfile(hhblits_prefix+".a3m") or \
     not os.path.isfile(hhblits_prefix+".aln"):
    logger.warning("Using single sequence for hmmsearch+hhblits")
    return run_hhblits(query_fasta, db, ncpu, hmmsearch_prefix)

  hmsblits_nf = run_hhblits(hhblits_prefix + ".a3m", db, ncpu, hmmsearch_prefix)

  ### remove redundancy in resulting msa ###
  if (filter_hmsblits==1 and hmsblits_nf>=target_nf[0]) or \
      filter_hmsblits==2:
    ### filter $hmmsearch_prefix.aln ###
    cmd = rmRedundantSeq_template.substitute(
        id_cut=id_cut[-1],  # typically the same or more stringent
        # than that in hhblits
        cov_cut=cov_cut[0],
        infile1=hhblits_prefix + ".aln",
        infile2=hmmsearch_prefix + ".aln",
        outprefix=hmmsearch_prefix,
    )
    logger.info(cmd)
    os.system(cmd)

    ### update $hmmsearch_prefix.a3m accordingly ###
    txt = ""
    with open(hmmsearch_prefix + ".aln", "r") as fp:
      aln_list = fp.read().splitlines()

    with open(hhblits_prefix + ".a3m", "r") as fp_hhba3m:
      with open(hmmsearch_prefix + ".a3m", "r") as fp_hmsa3m:
        for block in ("\n" + fp_hhba3m.read() + fp_hmsa3m.read()).split("\n>"):
          lines = block.splitlines()
          if len(lines) != 2:
            continue
          header, sequence = lines
          match_seq = sequence.translate(None, ascii_lowercase)
          if match_seq in aln_list:
            txt += f">{header}\n{sequence}\n"
            for l in range(len(aln_list)):
              if match_seq == aln_list[l]:
                del aln_list[l]
                break

    with open(hmmsearch_prefix + ".a3m", "w") as fp:
      fp.write(txt)
    hmsblits_nf = get_neff(hmmsearch_prefix)
  return hmsblits_nf


def search_metaclust(query_fasta, sequence, hhblits_prefix, db_list, ncpu,  # pylint: disable=redefined-outer-name
                     hmmsearch_prefix):
  ''' jump start hmmsearch using hhblits or jack_hhblits a3m'''

  #### convert input to hmm ####
  if aln2hmm:
    input_aln = hhblits_prefix + ".aln"
    if not os.path.isfile(input_aln):
      logger.warning(
          "No hhblits alignment. Using query sequence as input for hmmsearch")
      with open(input_aln, "w") as fp:
        fp.write(sequence + "\n")

    cmd = qhmmbuild_aln_template.substitute(
        infile=input_aln,
        outprefix=hmmsearch_prefix,
    )
  else:
    input_a3m = hhblits_prefix + ".a3m"
    if not os.path.isfile(input_a3m):
      logger.warning(
          "No hhblits alignment. Using query sequence as input for hmmsearch")
      with open(input_a3m, "w") as fp:
        fp.write(">seq\n" + sequence + "\n")

    cmd = qhmmbuild_a3m_template.substitute(
        infile=input_a3m,
        outprefix=hmmsearch_prefix,
    )
  logger.info(cmd)
  os.system(cmd)

  if not build_hmmsearch_db:
    return run_hmmsearch(query_fasta, sequence, hhblits_prefix, db_list, ncpu,
                         hmmsearch_prefix)
  else:
    return run_hmsblits(query_fasta, sequence, hhblits_prefix, db_list, ncpu,
                        hmmsearch_prefix)


def build_msa(prefix,  # pylint: disable=redefined-outer-name
              sequence,  # pylint: disable=redefined-outer-name
              tmpdir,  # pylint: disable=redefined-outer-name
              db_dict,  # pylint: disable=redefined-outer-name
              ncpu=1,
              overwrite=0,
              early_stopping=-1):
  ''' sequentially attempt to build MSA by hhblits, jackhmmer+hhblits,
    and hmmsearch. '''
  neff_dict = {}
  def neff_float_sort_key(x):
    return 0 if math.isnan(x) else x
  def neff_dict_write(neff_dict):
    with open(f"{prefix}_neff_debug.json", "w") as f:
      f.write(json.dumps(neff_dict))

  #### preparing query ####
  query_fasta = os.path.join(tmpdir, "seq.fasta")  # pylint: disable=redefined-outer-name
  with open(query_fasta, "w") as fp:
    fp.write(f">seq\n{sequence}\n")

  #### run hhblits ####
  hhblits_prefix = os.path.join(tmpdir, "hhblits")
  if "hhblitsdb" in db_dict:
    if test_bitwise_option(overwrite, "hhblits") or not os.path.isfile(
        prefix + ".hhbaln") or not os.path.isfile(prefix + ".hhba3m"):
      # generates hhblits_prefix.a3m hhblits_prefix.aln
      # hhblits_prefix.60.a3m hhblits_prefix.60.aln
      nf = run_hhblits(query_fasta, db_dict["hhblitsdb"], ncpu, hhblits_prefix)
      shutil.copyfile(hhblits_prefix + ".aln", prefix + ".hhbaln")
      shutil.copyfile(hhblits_prefix + ".a3m", prefix + ".hhba3m")
    else:
      shutil.copyfile(prefix + ".hhbaln", hhblits_prefix + ".aln")
      shutil.copyfile(prefix + ".hhba3m", hhblits_prefix + ".a3m")
      nf = get_neff(hhblits_prefix)
      logger.info("%s.{hhbaln,hhba3m} exists, skip hhblitsdb", prefix)

    neff_dict[hhblits_prefix] = nf
    if nf >= target_nf[0] and test_bitwise_option(early_stopping, "hhblits"):
      neff_dict_write(neff_dict)
      shutil.copyfile(hhblits_prefix + ".aln", prefix + ".aln")
      logger.info("Final MSA by hhblits with Nf >= %.1f", nf)
      return nf

  #### run jack_hhblits ####
  jackblits_prefix = os.path.join(tmpdir, "jackblits")
  if "jackhmmerdb" in db_dict:
    if test_bitwise_option(overwrite, "jackhmmer") or not os.path.isfile(
        prefix + ".jacaln") or not os.path.isfile(prefix + ".jaca3m"):
      # generates jackblits_prefix.a3m jackblits_prefix.aln
      # jackblits_prefix.60.a3m jackblits_prefix.60.aln
      nf = run_jackblits(query_fasta, db_dict["jackhmmerdb"], ncpu,
                         hhblits_prefix, jackblits_prefix)
      shutil.copyfile(jackblits_prefix + ".aln", prefix + ".jacaln")
      shutil.copyfile(jackblits_prefix + ".a3m", prefix + ".jaca3m")
    else:
      shutil.copyfile(prefix + ".jacaln", jackblits_prefix + ".aln")
      shutil.copyfile(prefix + ".jaca3m", jackblits_prefix + ".a3m")
      nf = get_neff(jackblits_prefix)
      logger.info("%s.{jacaln,jaca3m} exists, skip jackhmmerdb", prefix)

    neff_dict[jackblits_prefix] = nf
    if nf >= target_nf[0] and test_bitwise_option(early_stopping, "jackhmmer"):
      neff_dict_write(neff_dict)
      shutil.copyfile(jackblits_prefix + ".aln", prefix + ".aln")
      logger.info("Final MSA by jackhmmer with Nf >= %.1f", nf)
      return nf

  #### run bfdsearch ####
  bfd_prefix = os.path.join(tmpdir, "bfd")
  if "bfddb" in db_dict:
    if test_bitwise_option(overwrite, "bfd") or not os.path.isfile(
        prefix + ".bfdaln") or not os.path.isfile(prefix + ".bfda3m"):
      nf = run_bfd(query_fasta, db_dict["bfddb"], ncpu, hhblits_prefix,
                   jackblits_prefix, bfd_prefix)
      shutil.copyfile(bfd_prefix + ".aln", prefix + ".bfdaln")
      shutil.copyfile(bfd_prefix + ".a3m", prefix + ".bfda3m")
    else:
      shutil.copyfile(prefix + ".bfdaln", bfd_prefix + ".aln")
      shutil.copyfile(prefix + ".bfda3m", bfd_prefix + ".a3m")
      nf = get_neff(bfd_prefix)
      logger.info("%s.{bfdaln,bfda3m} exists, skip bfddb", prefix)
    neff_dict[bfd_prefix] = nf
    if nf >= target_nf[0] and test_bitwise_option(early_stopping, "bfd"):
      neff_dict_write(neff_dict)
      shutil.copyfile(bfd_prefix + ".aln", prefix + ".aln")
      shutil.copyfile(bfd_prefix + ".a3m", prefix + ".a3m")
      logger.info("Final MSA by bfd with Nf >= %.1f", nf)
      return nf

  #### run hmmsearch ####
  hmmsearch_prefix = os.path.join(tmpdir, "hmmsearch")
  if "hmmsearchdb" in db_dict:
    if test_bitwise_option(overwrite, "hmmsearch") or \
        not os.path.isfile(prefix + ".hmsaln") or \
        (build_hmmsearch_db and not os.path.isfile(prefix + ".hmsa3m")):
      # generates hmmsearch_prefix.afq hmmsearch_prefix.hmm
      # hmmsearch_prefix.redundant hmmsearch_prefix.nonredundant
      # hmmsearch_prefix.aln
      if neff_dict:
        meta_prefix, _ = max(neff_dict.items(),
                             key=lambda x: neff_float_sort_key(x[1]))
      else:
        meta_prefix = hhblits_prefix
      nf = search_metaclust(query_fasta, sequence, meta_prefix,
                            db_dict["hmmsearchdb"], ncpu, hmmsearch_prefix)
      shutil.copyfile(hmmsearch_prefix + ".aln", prefix + ".hmsaln")
      if os.path.isfile(hmmsearch_prefix + ".a3m"):
        shutil.copyfile(hmmsearch_prefix + ".a3m", prefix + ".hmsa3m")
    else:
      shutil.copyfile(prefix + ".hmsaln", hmmsearch_prefix + ".aln")
      if os.path.isfile(prefix + ".hmsa3m"):
        shutil.copyfile(prefix + ".hmsa3m", hmmsearch_prefix + ".a3m")
      logger.info("%s.hmsaln exists, skip hmmsearchdb", prefix)
      nf = get_neff(hmmsearch_prefix)

    neff_dict[hmmsearch_prefix] = nf
    if nf >= target_nf[0] and test_bitwise_option(early_stopping, "hmmsearch"):
      # hmmsearch replaces jackblits and hhblits result
      neff_dict_write(neff_dict)
      shutil.copyfile(hmmsearch_prefix + ".aln", prefix + ".aln")
      shutil.copyfile(hmmsearch_prefix + ".a3m", prefix + ".a3m")
      logger.info("Final MSA by hmmsearch with Nf >= %.1f", nf)
      return nf

  neff_dict_write(neff_dict)
  logger.debug("### Neff summary ###")
  for db_prefix, nf in neff_dict.items():
    logger.debug("%s MSA has %.1f Nf.", db_prefix, nf)

  db_prefix, nf = max(neff_dict.items(),
                      key=lambda x: neff_float_sort_key(x[1]))
  shutil.copyfile(db_prefix + ".aln", prefix + ".aln")
  shutil.copyfile(db_prefix + ".a3m", prefix + ".a3m")
  logger.info("%s MSA has %.1f Nf. Output anyway.", db_prefix, nf)
  return nf


def test_bitwise_option(option, *keys):
  ''' whether each bit is set in option. '''
  option_bits = {"hhblits": 1, "jackhmmer": 2, "bfd": 4, "hmmsearch": 8}
  assert option < (1<<4)
  if all(option & option_bits[key] for key in keys):
    return True
  return False


def refilter_aln(prefix, tmpdir):  # pylint: disable=redefined-outer-name
  ''' filter final MSA by -id 99 -cov 60 '''
  #### filter MSA ####
  final_prefix = os.path.join(tmpdir, "final")
  shutil.copyfile(prefix + ".aln", final_prefix + ".aln")
  cov = cov_cut[1]
  cmd = alnfilter_template.substitute(
      id_cut=id_cut[0],
      cov_cut=cov,
      prefix=final_prefix,
  )
  logger.info(cmd)
  os.system(cmd)
  shutil.copyfile(final_prefix + f".{cov}.aln", prefix + ".aln")

  #### calculate Nf ####
  cmd = calNf_template.substitute(
      infile=final_prefix + f".{cov}.aln",
      target_nf=target_nf[-1],
  )
  with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as p:
    stdout, _ = p.communicate()
  nf = float(stdout)
  logger.info("Re-filter final MSA to Nf >= %.1f", nf)
  return nf


if __name__ == "__main__":
  import argparse

  class SmartArgumentHelpFormatter(argparse.ArgumentDefaultsHelpFormatter):
    """Format epilog customly"""
    def _fill_text(self, text, width, indent):
      magic, sep = "R|", " - "
      def _split_kv(line):
        k = line.find(sep)
        if k == -1:
          return None, line
        return line[:k], line[k + len(sep):]
      if text.startswith(magic):
        rebrokenstr = []
        for k, v in map(_split_kv, text[len(magic):].splitlines()):
          self_indent = len(v) - len(v.lstrip())
          tlinearr = self._split_lines(v, width)
          if tlinearr:
            if exists(k):
              rebrokenstr.append(f"{k}{sep}{tlinearr[0]}")
              new_indent = indent + " "*len(k) + " "*len(sep)
            else:
              new_indent = indent + " "*self_indent
              rebrokenstr.append(f"{new_indent}{tlinearr[0]}")
            for tlinepiece in tlinearr[1:]:
              rebrokenstr.append(f"{new_indent}{tlinepiece}")
          else:
            if exists(k):
              rebrokenstr.append(f"{k}{sep}")
            else:
              rebrokenstr.append("")
        return "\n".join(rebrokenstr)
      return super()._fill_text(text, width, indent)

  #### command line argument parsing ####
  parser = argparse.ArgumentParser(
      formatter_class=SmartArgumentHelpFormatter,
      # pylint: disable=line-too-long
      epilog="R|"
             "output:\n"
             "  (filename prefix determined by input filename)\n"
             "  seq.aln    - final alignment. the only non-optional output\n"
             "  seq.hhbaln - (if --hhblitsdb is set) hhblits MSA (PSICOV format)\n"
             "  seq.hhba3m - (if --hhblitsdb is set) hhblits MSA (a3m format)\n"
             "  seq.jacaln - (if --jackhmmerdb is set and --hhblitsdb search does not have"
             "                enough sequences) jackhmmer + hhblits MSA (PSICOV format)\n"
             "  seq.jaca3m - (if --jackhmmerdb is set and --hhblitsdb search does not have"
             "                enough sequences) jackhmmer + hhblits MSA (a3m format)\n"
             "  seq.bfdaln - (if --bfddb is set and neither --hhblitsdb nor --jackhmmerdb"
             "                search has enough sequences)+ hhblits MSA (PSICOV format)\n"
             "  seq.bfda3m - (if --bfddb is set and neither --hhblitsdb nor --jackhmmerdb"
             "                search has enough sequences) hhblits MSA (a3m format)\n"
             "  seq.hmsaln - (if --hmmsearchdb is set and neither --hhblitsdb nor"
             "                --jackhmmerdb search has enough sequences)"
             "                hhblits + jackhmmer (optional) + hmmsearch output\n")
      # pylint: enable=line-too-long

  parser.add_argument("fasta_file", type=str, default=None,
      help="fasta file")
  parser.add_argument("--hhblitsdb", type=str, default=None,
      help="hhsuite databases, to be searched by hhblits.")
  parser.add_argument("--jackhmmerdb", type=str, default=None, nargs="+",
      help="(decompressed) fasta databases, to be searched by jackhmmer, whose"
           "search result will be built into a custom hhsuite database to be"
           "searched by hhblits, jump-starting from alignment generated by"
           "searching hhblitsdb. fasta database must have an ssi index file"
           "created by esl-sfetch.")
  parser.add_argument("--bfddb", type=str, default=None, nargs="+",
      help="BFD databases, to be searched by hhblits.")
  parser.add_argument("--hmmsearchdb", type=str, default=None, nargs="+",
      help="blank deliminated list of decompressed fasta database, to be"
           "searched hmmsearch, jump-starting from alignment generated by"
           "searching either --hhblitsdb or --jackhmmerdb. fasta database must"
           "have an ssi index file created by esl-sfetch.")
  parser.add_argument("--tmpdir", type=str, default="",
      help="temporary forlder. default=/tmp/$USER/MSA_`date +%%N`")
  parser.add_argument("-k", "--keep", action="store_true",
      help="keep (don\'t delete) temporary files")
  parser.add_argument("-o", "--outdir", type=str, default=".",
      help="output forlder. default is current folder")
  parser.add_argument("--ncpu", type=int, default=1,
      help="number of CPU threads. do not use multi-threading by default.")
  parser.add_argument("--overwrite", type=int, default=0,
      help="whether overwrite existing search result.\n"
           "0 - do not overwrite any intermediate alignment"
           "1 - overwrite hhblitsdb search result (.hhbaln and .hhba3m)"
           "2 - overwrite jackhmmerdb search result (.jacaln and .jaca3m)"
           "4 - overwrite bfddb search result (.bfdaln and .bfda3m)"
           "8 - overwrite hmmsearchdb search result (.hmsaln)"
           "These options are addictive. For example, --overwrite=15 (1+2+4+8)"
           "for overwriting any intermediate alignment (but might still filter"
           "final alignment if it is too large).")
  parser.add_argument("--early_stopping", type=int, default=-1,
      help="whether early stop the searching."
           "0 - do not early stop searching"
           "1 - early stopping hhblitsdb search (.hhbaln and .hhba3m)"
           "2 - early stopping jackhmmerdb search (.jacaln and .jaca3m)"
           "4 - early stopping bfddb search (.bfdaln and .bfda3m)"
           "8 - early stopping hmmsearchdb search (.hmsaln)"
           "These options are addictive. For example, "
           "--early_stopping=15 (1+2+4+8) for stopping if Neff is large "
           "enough. ")
  parser.add_argument("-v", "--verbose", action="store_true", help="verbose")

  args = parser.parse_args()

  db_dict = {}
  if args.hhblitsdb:
    db_dict["hhblitsdb"] = os.path.abspath(args.hhblitsdb)
  if args.jackhmmerdb:
    db_dict["jackhmmerdb"] = list(map(os.path.abspath, args.jackhmmerdb))
  if args.bfddb:
    db_dict["bfddb"] = list(map(os.path.abspath, args.bfddb))
  if args.hmmsearchdb:
    db_dict["hmmsearchdb"] = list(map(os.path.abspath, args.hmmsearchdb))

  if all(map(lambda x: not x, db_dict.values())):
    parser.print_help()
    sys.exit()

  #### check input format ####
  if not os.path.isfile(args.fasta_file):
    print(f"ERROR! No such query fasta {args.fasta_file}\n")
    sys.exit()

  logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)

  check_db(db_dict)
  sequence = read_one_sequence(args.fasta_file)
  tmpdir = make_tmpdir(args.tmpdir, prefix="MSA_")
  prefix = os.path.splitext(args.fasta_file)[0]
  if args.outdir and args.outdir != ".":
    mkdir_if_not_exist(args.outdir)
    prefix = os.path.join(args.outdir, os.path.basename(prefix))

  #### start building MSA ####
  neff = build_msa(prefix,
                   sequence,
                   tmpdir,
                   db_dict,
                   ncpu=args.ncpu,
                   overwrite=args.overwrite,
                   early_stopping=args.early_stopping)

  #### filter final MSA if too large ####
  # this will not improve contact accuracy. it is solely for making the
  # MSA not too large so that it is manageable for contact prediction
  if neff >= target_nf[-1]:
    neff = refilter_aln(prefix, tmpdir)

  ### clean up ####
  if os.path.isdir(tmpdir) and not args.keep:
    shutil.rmtree(tmpdir)