diff --git a/app.py b/app.py deleted file mode 100644 index 26ceb68..0000000 --- a/app.py +++ /dev/null @@ -1,40 +0,0 @@ -import secrets - -from flask import Flask, flash, redirect, render_template, request, session, url_for -from markupsafe import escape - -from src import host_lookup -import upload - -app = Flask(__name__) -generate_secret = secrets.token_urlsafe(16) -app.secret_key = generate_secret - - -@app.route("/") -def index(): - return redirect(url_for("lookup")) - - -@app.route("/lookup", methods=["GET", "POST"]) -def lookup(): - hosts = [] - results = [] - if request.method == "GET": - return render_template("lookup.html") - elif request.method == "POST" and "host" in request.form: - user_input = escape(request.form.get("host").strip()) - hosts, errors = host_lookup.process_input(user_input) - for host in hosts: - result = host_lookup.Lookedup(host) - results.append(result) - return render_template("results.html", hosts=results, errors=errors) - elif request.method == "POST" and "file" in request.files: - file = request.files["file"] - extracted = upload.extract(file) - hosts = host_lookup.process_file(extracted) - return render_template("results.html", hosts=hosts) - - -if __name__ == "__main__": - app.run(debug=True) diff --git a/constants.py b/constants.py deleted file mode 100644 index c7c46e5..0000000 --- a/constants.py +++ /dev/null @@ -1,30 +0,0 @@ -LOGCONF = { - "version": 1, - "formatters": { - "default": { - "format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s", - } - }, - "handlers": { - "console": { - "class": "logging.StreamHandler", - "stream": "ext://sys.stdout", - "formatter": "default", - }, - "file": { - "class": "logging.FileHandler", - "filename": "flask.log", - "formatter": "default", - }, - }, - "root": {"level": "DEBUG", "handlers": ["console", "file"]}, -} - -EMAIL = "email address" -URL = "url" -DOMAIN = "domain" -IPV4 = "ipv4" -IPV6 = "ipv6" -NO_HOST = "no host" - -domain_lookup = {} diff --git a/host_data.py b/host_data.py deleted file mode 100644 index a67fa27..0000000 --- a/host_data.py +++ /dev/null @@ -1,43 +0,0 @@ -from ipaddress import ip_address -from checkdmarc.dmarc import check_dmarc -from checkdmarc.spf import check_spf -import validators -from ipwhois import IPWhois -from whois import whois -from constants import DOMAIN, EMAIL, IPV4, IPV6, URL - - -def determine(host): - host_type = "" - if validators.url(host): - host_type = URL - elif validators.domain(host): - host_type = DOMAIN - elif validators.ip_address.ipv4(host): - host_type = IPV4 - elif validators.ip_address.ipv6(host): - host_type = IPV6 - elif validators.email(host): - host_type = EMAIL - else: - print("NO HOST TYPE") - return host_type - - -def domain(host): - result = dict(whois(host)) - if type(result["domain_name"]) is list: - result["domain_name"] = result["domain_name"][0] - return result - - -def emailsec(host): - spf = "" - dmarc = "" - result_spf = check_spf(host) - if result_spf["valid"]: - spf = result_spf["record"] - result_dmarc = check_dmarc(host) - if result_dmarc["valid"]: - dmarc = result_dmarc["record"] - return spf, dmarc diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 2d657d8..0000000 --- a/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[metadata] -name = my_soc_site -version = 0.9 -author = Joost Agterhoek -description = Flask project to learn Python and single-page websites -author_email = joostagterhoek@protonmail.com diff --git a/setup.py b/setup.py deleted file mode 100644 index 7af7be0..0000000 --- a/setup.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/python3 - -from setuptools import setup - -if __name__ == "__main__": - setup() diff --git a/src/abuseipdb_api.py b/src/abuseipdb_api.py deleted file mode 100644 index df6260f..0000000 --- a/src/abuseipdb_api.py +++ /dev/null @@ -1,52 +0,0 @@ -from base64 import decode -import json -import os -import requests -import requests_cache -from dotenv import load_dotenv -from pprint import pprint - - -class API_error(Exception): - pass - - -def environment(): - requests_cache.install_cache(expire_after=360, allowable_methods=("POST")) - load_dotenv() - api_key = os.getenv("ABUSEIPDB_API") - return api_key - - -def lookup(api_key, host): - url = "https://api.abuseipdb.com/api/v2/check" - payload = {"ipAddress": "", "maxAgeInDays": "90"} - payload.update({"ipAddress": host}) - headers = {"Accept": "application/json", "Key": api_key} - response = requests.request( - method="GET", url=url, params=payload, headers=headers, verify=False - ) # TODO: remove SSL verify=False and add signed certificate if possible. - # Figure out how caching functions here: https://requests-cache.readthedocs.io/en/stable/examples.html - response_dict = json.loads(response.text) - lookup = dict.fromkeys( - ["score", "last_reported", "IP_address", "CDN", "Tor", "total_reports"] - ) - print(response_dict) - lookup["score"] = response_dict["data"]["abuseConfidenceScore"] - lookup["last_reported"] = response_dict["data"]["lastReportedAt"] - lookup["IP_address"] = response_dict["data"]["ipAddress"] - lookup["usage"] = response_dict["data"]["usageType"] - lookup["Tor"] = response_dict["data"]["isTor"] - lookup["total_reports"] = response_dict["data"]["totalReports"] - - print(requests_cache.get_cache()) - print("Cached:") - print("\n".join(requests_cache.get_cache().urls())) - - return lookup - - -def analyse(host): - api_key = environment() - result = lookup(api_key, host) - return result diff --git a/src/host_lookup.py b/src/host_lookup.py deleted file mode 100644 index 3741df6..0000000 --- a/src/host_lookup.py +++ /dev/null @@ -1,154 +0,0 @@ -# TODO: make this module only have wrapper functions, no direct lookups or imports. Move all the direct lookup functions (emailsec) into (a) separate module(s). - -import re -from ipaddress import ip_address -from checkdmarc.dmarc import check_dmarc -from checkdmarc.spf import check_spf -import validators -from ipwhois import IPWhois -from whois import whois - -# from constants import DOMAIN, EMAIL, IPV4, IPV6, URL -from src import abuseipdb_api, virustotal_api -import socket -from urllib.parse import urlparse -import host_data -import tldextract - - -class Lookedup(object): - def __init__(self, host): - self.host = host - self.host_type = determine(self.host) - self = self.specific() - # TODO: consolidate all below functions if possible - - def url_lookup(self): - self.domain = urlparse(self.host).netloc - self.ip_address = socket.gethostbyname(self.domain) - self.metadata = domain(self.domain) - self.email_security = spf_dmarc(self.domain) - self.vt, self.vt_dict = virustotal_api.analyse2(self.host, self.host_type) - self.abuseipdb = abuseipdb_api.analyse(self.ip_address) - return self - - def ip_lookup(self): - self.metadata = domain(self.host) - self.domain = self.metadata["domain_name"] - self.email_security = spf_dmarc(self.domain) - self.vt, self.vt_dict = virustotal_api.analyse2(self.host, self.host_type) - self.abuseipdb = abuseipdb_api.analyse(self.host) - return self - - def domain_lookup(self): - self.ip_address = socket.gethostbyname(self.host) - self.metadata = domain(self.host) - self.domain = self.metadata["domain_name"] - self.email_security = spf_dmarc(self.domain) - self.vt, self.vt_dict = virustotal_api.analyse2(self.host, self.host_type) - self.abuseipdb = abuseipdb_api.analyse(self.ip_address) - return self - - def email_lookup(self): - self.domain = self.host.split("@")[1] - self.metadata = domain(self.domain) - self.ip_address = socket.gethostbyname(self.domain) - self.email_security = spf_dmarc(self.domain) - self.vt, self.vt_dict = virustotal_api.analyse2(self.domain, self.host_type) - self.abuseipdb = abuseipdb_api.analyse(self.ip_address) - return self - - def specific(self): - if self.host_type == "url": - return self.url_lookup() - elif self.host_type == "domain": - return self.domain_lookup() - elif self.host_type == "ip": - return self.ip_lookup() - elif self.host_type == "email address": - return self.email_lookup() - - -def sanitize(user_input): - sanitized = [] - if user_input.strip() != "": - sanitized = re.split("; |, | |\n", user_input) - return sanitized - - -def determine(host): - host_type = "" - if validators.url(host): - host_type = "url" - elif validators.domain(host): - host_type = "domain" - elif validators.ip_address.ipv4(host): - host_type = "ip" - elif validators.ip_address.ipv6(host): - host_type = "ip" - elif validators.email(host): - host_type = "email address" - else: - host_type = "no host" - return host_type - - -def extract(user_input): - hosts = [] - errors = [] - for item in user_input: - if validators.url(item): - hosts.append(item) - elif validators.domain(item): - hosts.append(item) - elif validators.ip_address.ipv4(item): - hosts.append(item) - elif validators.ip_address.ipv6(item): - hosts.append(item) - elif validators.email(item): - hosts.append(item) - else: - errors.append(item) - return hosts, errors - - -def domain(host): - result = dict(whois(host)) - if type(result["creation_date"]) is list: - result["creation_date"] = result["creation_date"][0].strftime("%d-%m-%Y") - else: - result["creation_date"] = result["creation_date"].strftime("%d-%m-%Y") - if type(result["domain_name"]) is list: - result["domain_name"] = result["domain_name"][0] - return result - - -def spf_dmarc(domain): - spf = "" - dmarc = "" - result_spf = check_spf(domain) - if result_spf["valid"]: - spf = result_spf["record"] - result_dmarc = check_dmarc(domain) - if result_dmarc["valid"]: - dmarc = result_dmarc["record"] - return spf, dmarc - - -def process_input(user): - results = [] - sanitized = sanitize(user) - hosts, errors = extract(sanitized) - # for host in hosts: - # host_analyzed = Host(host).lookup() - # results.append(host_analyzed) - # return results - return hosts, errors - - -def process_file(file_content): - hosts = [] - for host in file_content: - result = Host(host) - hosts.append(result) - return hosts diff --git a/src/virustotal_api.py b/src/virustotal_api.py deleted file mode 100644 index 6cd75a8..0000000 --- a/src/virustotal_api.py +++ /dev/null @@ -1,111 +0,0 @@ -import json -import time -import os -import requests -from dotenv import load_dotenv -from constants import URL, DOMAIN, IPV4, IPV6, domain_lookup - - -def environment(): - load_dotenv() - api_key = os.getenv("VT_API") - return api_key - - -def analysis_object(api_key, host): - url = "https://www.virustotal.com/api/v3/urls" - payload = {"url": ""} - payload.update({"url": host}) - headers = { - "accept": "application/json", - "content-type": "application/x-www-form-urlencoded", - "x-apikey": api_key, - } - response = requests.post(url, data=payload, headers=headers) - response_dict = json.loads(response.text) - response_id = response_dict["data"]["id"] - return response_id - - -def analyse_domain(api_key, host): - url = "https://www.virustotal.com/api/v3/domains/" + host - headers = { - "accept": "application/json", - "content-type": "application/x-www-form-urlencoded", - "x-apikey": api_key, - } - analysis_json = requests.get(url, headers=headers) - response_dict = json.loads(analysis_json.text) - return response_dict, analysis_json - - -def analyse_URL(api_key, response_id): - url = "https://www.virustotal.com/api/v3/analyses/{}".format(response_id) - headers = {"accept": "application/json", "x-apikey": api_key} - analysis_json = requests.get(url, headers=headers) - analysis_dict = json.loads(analysis_json.text) - return analysis_dict, analysis_json - - -# This returns a differently shaped JSON and therefore dict: -# analysis_dict keys 'data', 'meta' -# analysis_dict['data'] keys 'id', 'type', 'links', 'attributes' -# analysis_dict['data']['attributes'] keys 'stats', (numbers) 'results', (all the AV engine results) 'date', (Linux epoch timestamp) 'status' - - -def analyse_IP(api_key, host): - analysis_url = "https://www.virustotal.com/api/v3/ip_addresses/{}".format(host) - headers = {"accept": "application/json", "x-apikey": api_key} - analysis_json = requests.get(analysis_url, headers=headers) - response_dict = json.loads(analysis_json.text) - # Implement this: https://docs.virustotal.com/reference/ip-info - return response_dict, analysis_json - - -def analyse(host, host_type): - api_key = environment() - if host_type == URL: - response_id = analysis_object(api_key, host) - result, analysis_json = analyse_URL(api_key, response_id) - elif host_type == DOMAIN: - result, analysis_json = analyse_domain(api_key, host) - elif host_type == IPV4 or IPV6: - result, analysis_json = analyse_IP(api_key, host) - return result, analysis_json - - -def analyse2(host, host_type): - api_key = environment() - if host_type == "url": - response_id = analysis_object(api_key, host) - result, analysis_json = analyse_URL(api_key, response_id) - elif host_type == "domain" or host_type == "email address": - result, analysis_json = analyse_domain(api_key, host) - elif host_type == "ip": - result, analysis_json = analyse_IP(api_key, host) - if host_type == "url": - vt_stats = result["data"]["attributes"]["stats"] - vt_results = result["data"]["attributes"]["results"] - last_update = result["data"]["attributes"]["date"] - elif host_type == "domain" or host_type == "email address" or host_type == "ip": - vt_stats = result["data"]["attributes"]["last_analysis_stats"] - vt_results = result["data"]["attributes"]["last_analysis_results"] - last_update = result["data"]["attributes"]["last_analysis_date"] - - summary = dict.fromkeys(["total", "score", "vendors", "last_update"]) - total = 0 - vendors = [] - for key, value in vt_stats.items(): - total += value - for key, value in vt_results.items(): - if value["category"] == "malicious": - vendors.append(key) - - summary["total"] = total - summary["score"] = vt_stats["malicious"] - summary["vendors"] = vendors - summary["last_update"] = time.strftime( - "%d-%m-%Y", - time.gmtime(last_update), - ) - return summary, analysis_json diff --git a/static/javascript.js b/static/javascript.js deleted file mode 100644 index 73cdfca..0000000 --- a/static/javascript.js +++ /dev/null @@ -1,12 +0,0 @@ -$(function() { - $("td[colspan=7]").find("p").hide(); - $("table").click(function(event) { - event.stopPropagation(); - var $target = $(event.target); - if ( $target.closest("td").attr("colspan") > 1 ) { - $target.slideUp(); - } else { - $target.closest("tr").next().find("p").slideToggle(); - } - }); -}); diff --git a/static/styles/style.css b/static/styles/style.css deleted file mode 100644 index b8b6a44..0000000 --- a/static/styles/style.css +++ /dev/null @@ -1,179 +0,0 @@ -@import url('https://fonts.googleapis.com/css2?family=DM+Mono:wght@300;400;500&display=swap'); - -body { - min-height: 100vh; - padding: 0; - margin: 0; - background-color: #17141d; - color: white; - font-family: 'DM Mono', monospace; -} - -.headings { - color: #fdc105; - display: flex; - justify-content: center; -} - -.input_container { - padding: 30px 30px; - display: flex; - justify-content: center; -} - -.forms_container { - display: flex; - justify-content: around; - align-items: center; - flex-direction: column; - padding: 75px 250px; - color: #fff; - background-image: url(/static/background/cool-background-trianglify.png); - /* background: linear-gradient(to bottom right, red, yellow); */ - border: 0; - transition: 0.2s; - overflow: hidden; -} - -.input_form { - padding: 5px 5px; - display: flex; -} - -.upload_form { - padding: 5px 5px; - display: flex; -} - -.btn-warning { - position: relative; - padding: 11px 16px; - font-size: 15px; - line-height: 1.5; - border-radius: 3px; - color: #000000; - background-color: #ffc100; - border: 0; - transition: 0.2s; - opacity: 50; - overflow: hidden; - } - -.btn-warning input[type="file"] { - color: transparent; - cursor: pointer; - position: absolute; - left: 0%; - top: 0%; - transform: scale(3); - opacity: 50; - transition: 0.2s; - } -.btn-warning:hover { - background-color: #eed519; - } - -.submit-button { - position: relative; - padding: 11px 16px; - font-size: 15px; - line-height: 1.5; - border-radius: 3px; - color: #000000; - background-color: #ffc100; - border: 0; - transition: 0.2s; - opacity: 50; - overflow: hidden; -} - -.submit-button:hover { - background-color: #eed519; -} - -.table-container { - padding: 50px 50px; - border-radius: 3px; -} - -.errors-container { - display: flex; - justify-content: center; - padding: 10px 10px; - border-radius: 3px; - -} - -.results-table { - padding: 50px 50px; - border-radius: 3px; -} - -#myTable { - background-color: #fad253; - border-collapse: collapse; - width: 100%; - border: 1px solid #ddd; - font-size: 14px; -} - -#myTable th { - background-color: #fad253; - text-align: left; /* Left-align text */ - padding: 12px; /* Add padding */ -} - -#myTable tr { - /* Add a bottom border to all table rows */ - border-bottom: 1px solid #ddd; -} - -/*#myTable tr.header,*/ #myTable tr:hover { - /* Add a grey background color to the table header and on hover */ - background-color: #ec974f; -} - -#myTable td[colspan] { - padding-top: 0; padding-bottom: 0; -} - -#myTable td[colspan] p { - padding-top: 12px; padding-bottom: 12px; -} - -/* TEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEST */ - - -#data { - background-color: #fad253; - border-collapse: collapse; - width: 100%; - border: 1px solid #ddd; - font-size: 14px; -} - -#data th { - background-color: #fad253; - text-align: left; /* Left-align text */ - padding: 12px; /* Add padding */ -} - -#data tr { - /* Add a bottom border to all table rows */ - border-bottom: 1px solid #ddd; -} - -/*#myTable tr.header,*/ #data tr:hover { - /* Add a grey background color to the table header and on hover */ - background-color: #ec974f; -} - -#data td[colspan] { - padding-top: 0; padding-bottom: 0; -} - -#data td[colspan] p { - padding-top: 12px; padding-bottom: 12px; -} - - diff --git a/style.css b/style.css deleted file mode 100644 index adc68fa..0000000 --- a/style.css +++ /dev/null @@ -1,3 +0,0 @@ -h1 { - color: red; -} diff --git a/templates/lookup.html b/templates/lookup.html deleted file mode 100644 index 0e5c958..0000000 --- a/templates/lookup.html +++ /dev/null @@ -1,12 +0,0 @@ -{% extends "layout.html" %} - - - -{% block content %} - -{% include "forms.html" %} - -{% include "empty_form.html" %} - -{% endblock %} - diff --git a/templates/results-detail.html b/templates/results-detail.html deleted file mode 100644 index e69de29..0000000 diff --git a/templates/results-overview-new.html b/templates/results-overview-new.html deleted file mode 100644 index 8b73068..0000000 --- a/templates/results-overview-new.html +++ /dev/null @@ -1,98 +0,0 @@ - - - -
- {% if error in errors %} -

- - You may have mistyped: {{errors|join(',')}}. Try again! - - {% endif %} -

- -
-
- - - - - - - - - - - - {% for host in hosts %} - - - - - - - - - - - - - {% endfor %} - -
IP addressVirusTotalAbuseIPDBDomainRegistration countryRegistration dateEmail security
{{host.host}}Score: {{host.vt['score']}} / {{host.vt['total']}}
- Last updated: {{host.vt['last_update']}}
- 		Score: {{host.abuseipdb['score']}} - Last reported: {{host.abuseipdb['last_reported']}}{{host.domain}}{{host.metadata['registrar_country']}}{{host.metadata['creation_date']}}SPF: {{host.email_security[0]}}, DMARC: {{host.email_security[1]}}
-

- VirusTotal -
- {% if host.vt['score'] > 0 %} - Vendor marked this host as malicious or suspicious: - {{host.vt['vendors']|join(', ')}} - {{host.vt['score']}} - {% else %} - No vendors marked this host as malicious or suspicious. - {% endif %} -
- AbuseIPDB -
- IP address is: {{host.abuseipdb['IP_address']}}
- Is this Tor: {{host.abuseipdb['Tor']}}
- Usage: {{host.abuseipdb['usage']}}
-
- -

-
-
-
- - - -{% block scripts %} - - - -{% endblock %} diff --git a/templates/results-overview.html b/templates/results-overview.html deleted file mode 100644 index 2573ee9..0000000 --- a/templates/results-overview.html +++ /dev/null @@ -1,87 +0,0 @@ - - - - -
-
- - - - - - - - - - - - {% for host in hosts %} - - - - - - - - - - - - - {% endfor %} - -
IP addressVirusTotalAbuseIPDBDomainRegistration countryRegistration dateEmail security
{{host.host}}Score: {{host.vt['vendors_malicious']}} / {{host.vt['vendors_total']}}
- Last updated: {{host.vt['last_update']}}		Score: {{host.abuseipdb['data']['abuseConfidenceScore']}} - Last reported: {{host.abuseipdb['data']['lastReportedAt']}}{{host.domain}}{{host.metadata['registrar_country']}}{{host.metadata['creation_date']}}SPF: {{host.emailsec[0]}}, DMARC: {{host.emailsec[1]}}
-

- Vendor marked this host as malicious or suspicious: - {% for key, value in host.vt['all_vendors'].items() if value['category'] in ['malicious', 'suspicious'] %} - {{key}} - {% endfor %} -

- -
-
-
- -{# {% include "test-table.html" %} #} - - -{% block scripts %} - -{#{% for host in hosts %}#} -{#whoopsie = JSON.parse({{ host.vt_json.text | tojson }}) {% endfor %}#} -{#{{whoopsie}}#} - - - -{% endblock %} diff --git a/templates/results.html b/templates/results.html deleted file mode 100644 index 9a17271..0000000 --- a/templates/results.html +++ /dev/null @@ -1,17 +0,0 @@ -{% extends "layout.html" %} - -{% block content %} - -{# For multiple results, maybe make a top results overview and then a navigation menu to specific URL pages? See: https://realpython.com/primer-on-jinja-templating/#include-a-navigation-menu #} - -{% include "forms.html" %} - - {# {% include "results-overview.html" %} #} - - {% include "results-overview-new.html" %} - -{% include "results-detail.html" %} - -{% include "empty_form.html" %} - -{% endblock %} diff --git a/upload/csv_parse.py b/upload/csv_parse.py deleted file mode 100644 index fce7543..0000000 --- a/upload/csv_parse.py +++ /dev/null @@ -1,22 +0,0 @@ -import csv -from io import StringIO -from validators import ipv4, ipv6, url, domain - - -def extract(uploaded): - hosts = [] - content = uploaded.read() - decoded = content.decode("utf-8") - file = StringIO(decoded) - csv_data = csv.reader(file, delimiter=",") - for row in csv_data: - for value in row: - if url(value): - hosts.append(value) - elif domain(value): - hosts.append(value) - elif ipv4(value): - hosts.append(value) - elif ipv6(value): - hosts.append(value) - return hosts