scylla.py.maintenance

#!/var/www/scylla/scyllaenv/bin/python
import sys
sys.path.insert(0, '/var/www/scylla')
sys.path.insert(0, '/var/www/scylla/scyllaenv/lib/python3.5/site-packages')
from flask import Flask, jsonify, request, render_template, url_for
app = Flask(__name__)
import subprocess
from sh import ls
from sh import head
import json
#from base58 import b58encode
import os
from hurry.filesize import size
from functools import wraps
from flask import request, Response
from operator import itemgetter
from json2html import json2html
import glob
import requests
from flask import jsonify
from werkzeug.utils import secure_filename

UPLOAD_FOLDER = "/var/www/scylla/user_uploaded_dbs/"

app = Flask(__name__)
app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER
app.config["MAX_CONTENT_PATH"] = 999999999999999999999999999999999999
#https://2e702ecae623463fb72c1fbf5e291bf7.us-west-2.aws.found.io:9243
#2kZN6fjkssx3Jy1fivLrQbcV

HOST = "https://c381c1ada274433ebcebc60f54520848.us-west-2.aws.found.io:9243"
USER = "elastic"
PASSWORD = "2kZN6fjkssx3Jy1fivLrQbcV"


def check_auth(username, password):
    """This function is called to check if a username /
    password combination is valid.
    """
    return username == 'sammy' and password == 'BasicPassword!'

def authenticate():
    """Sends a 401 response that enables basic auth"""
    return Response(
    '''Login Required - email acaceres@0-sec.net for access or DM him @_hyp3ri0n on Twitter. Please do so from a corporate email or
    with some kind of proof that you are a security engineer, academic, or need this data for research purposes (e.g. email from a corporate or .edu email or provide
    a linkedin or twitter account showing as much)
    You have to login with proper credentials''', 401, {'WWW-Authenticate': 'Basic realm="Login Required - email acaceres@0-sec.net for access or DM him @_hyp3ri0n on Twitter. Please do so from a corporate email or with some kind of proof that you are a security engineer, academic, or need this data for research purposes (e.g. email from a corporate or .edu email or provide a linkedin or twitter account showing as much)"'})

def requires_auth(f):
    @wraps(f)
    def decorated(*args, **kwargs):
        auth = request.authorization
        if not auth or not check_auth(auth.username, auth.password):
            return authenticate()
        return f(*args, **kwargs)
    return decorated

#@app.route("/status")
def check_status():

    _dir = ls("-alhS", "/var/www/")
#    print(_dir)
    return str(_dir)

#@app.route("/grab")
def grab():
    f = open("/var/www/html/results/" + request.args.get("wut"))
    return f.read()

#@app.route("/")
#TODO: TURNED OFF AUTH
#@requires_auth
def index():
    
    user_agent = request.headers.get('User-Agent').lower()

    if "wget" in user_agent or "curl" in user_agent or "aria" in user_agent:
        return "Using cli tools to download all databases is discouraged. You are encouraged to download in browser. If this is not possible I would simply ask that you open up a browser with scylla.sh (to allow cryptomining) while you do so. Note: this is a simple user-agent string check, to bypass this message simply falsify your user-agent header to a common browser." ,409



@app.route("/")
@app.route("/search")
def search():
    size = 100
    _params = {"q" : request.args.get("q", default="*:*").title(), "size" : request.args.get("size", default="20"), "from" : request.args.get("from", default=0), "q.parser" : "lucene"}
    #r = requests.get(HOST + "/pw_data/_search", params = _params,
    #                 auth = (USER, PASSWORD))
    
    #if ":" not in _params["q"]:
    #    return "<h1> 500 Internal Server Error.</h1> You do not appear to be using Solr/Lucene query syntax.", 500
    
    #print(_params)
    #print(r.url)
    #print("==============================")
    #print(r.text)
    #print(r.json)
    json_hits_num = 0
    #json_hits_raw = r.json()["hits"]["hits"]
    #print("==============================")

    #if 'Accept' in request.headers and request.headers['Accept'] == 'application/json':
    #    return jsonify(json_hits_raw)
    
    json_hits_filtered = []
    #print(json.dumps(json_hits_filtered))

    all_keys = []
    for hit in json_hits_filtered:
        all_keys += hit.keys()
    all_keys = list(set(all_keys))

    html = "<table id=\"results\">"
    html += "<tr>"
    for key in all_keys:
        html += "<th>" + key + "</th>"
    html += "</tr>"

    #TODO: this needs a helper function
    for hit in json_hits_filtered:
        html += "<tr>"
        for key in all_keys:
            print(key, hit)            
            try:
                html += "<th>" + hit[key] + "</th>"
            except:                
                html += "<th>" + "null" + "</th>"
        html += "</tr>"

    html += "</table>"
    
    #json_hits_html = json2html.convert(json = json.dumps(json_hits_filtered), table_attributes = 'id="search-table"')

    pages = int(json_hits_num/size)

    dbs = glob.glob("/home/ubuntu/bighd/normalized/05Aug_normed/*.jl")
    domains = []
    for jl_file in dbs:
        f = open(jl_file)
        for line in f:
            _dic = json.loads(line.strip())
            if "Domain" in _dic:
                domains.append(_dic["Domain"])
            break
    dbs = domains

    try:
        fields = []
    except KeyError:
        fields = []

    return render_template("index.html", params = _params, hits_num = json_hits_num, fields = fields, pages = range(0, pages), size = size,
                           dbs = dbs, html_results = html)
    
@app.route("/claim", methods=["POST"])
def claimed(db_name):

    claimed = ["orrent-invite", "cannabis.", "rambler", "ovh_kimsufi_2", "yande", "forbes", "twitter",
               "MyChemicalRomanc", "BlackMarketReloa", "Plex.tv", "honomaru.", "leetcc.7z",
               "bit.ly_9.3m_social_May", "nulled.io", "R2Games", "gawker_r", "Hostinger_Hacked", "geeks", "Habb"]

    for _item in claimed:
        if _item in db_name:
            return True

    return False


@app.route("/progress", methods = ["POST"])
def progress():

    f = secure_filename(request.files['file'])
    
    f.save("/var/www/scylla/user_uploaded_dbs/" + f)
    print(f)
    
    return str("Thanks for the db!")


@app.route("/crowdsource", methods = ["GET", "POST"])
def crowdsource():

    dbs_out = subprocess.run("ls -alhS /var/www/scylla/static/dbs | head -n 100", shell = True, stdout = subprocess.PIPE)
    dbs = dbs_out.stdout.decode("utf-8").split("\n")

    db_name = []
    for db in dbs:
        split = db.split()
        if len(split) > 2:
            if claimed(split[-1]):
                db_name.append((split[-1], split[4], "Claimed"))
            else:
                db_name.append((split[-1], split[4], "Unclaimed"))

    dbs = db_name
    return render_template("crowdsource.html", dbs = dbs)


if __name__ == "__main__":
    app.run(host = "0.0.0.0", threaded=True, port=8082, debug=True)