removing old code in favor of packaging-friendly setup

This commit is contained in:
Joost Agterhoek 2025-04-15 14:32:01 +02:00
parent c1f3b695a9
commit 5c2eefc17d
17 changed files with 0 additions and 872 deletions

40
app.py
View File

@ -1,40 +0,0 @@
import secrets
from flask import Flask, flash, redirect, render_template, request, session, url_for
from markupsafe import escape
from src import host_lookup
import upload
app = Flask(__name__)
generate_secret = secrets.token_urlsafe(16)
app.secret_key = generate_secret
@app.route("/")
def index():
return redirect(url_for("lookup"))
@app.route("/lookup", methods=["GET", "POST"])
def lookup():
hosts = []
results = []
if request.method == "GET":
return render_template("lookup.html")
elif request.method == "POST" and "host" in request.form:
user_input = escape(request.form.get("host").strip())
hosts, errors = host_lookup.process_input(user_input)
for host in hosts:
result = host_lookup.Lookedup(host)
results.append(result)
return render_template("results.html", hosts=results, errors=errors)
elif request.method == "POST" and "file" in request.files:
file = request.files["file"]
extracted = upload.extract(file)
hosts = host_lookup.process_file(extracted)
return render_template("results.html", hosts=hosts)
if __name__ == "__main__":
app.run(debug=True)

View File

@ -1,30 +0,0 @@
LOGCONF = {
"version": 1,
"formatters": {
"default": {
"format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s",
}
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"stream": "ext://sys.stdout",
"formatter": "default",
},
"file": {
"class": "logging.FileHandler",
"filename": "flask.log",
"formatter": "default",
},
},
"root": {"level": "DEBUG", "handlers": ["console", "file"]},
}
EMAIL = "email address"
URL = "url"
DOMAIN = "domain"
IPV4 = "ipv4"
IPV6 = "ipv6"
NO_HOST = "no host"
domain_lookup = {}

View File

@ -1,43 +0,0 @@
from ipaddress import ip_address
from checkdmarc.dmarc import check_dmarc
from checkdmarc.spf import check_spf
import validators
from ipwhois import IPWhois
from whois import whois
from constants import DOMAIN, EMAIL, IPV4, IPV6, URL
def determine(host):
host_type = ""
if validators.url(host):
host_type = URL
elif validators.domain(host):
host_type = DOMAIN
elif validators.ip_address.ipv4(host):
host_type = IPV4
elif validators.ip_address.ipv6(host):
host_type = IPV6
elif validators.email(host):
host_type = EMAIL
else:
print("NO HOST TYPE")
return host_type
def domain(host):
result = dict(whois(host))
if type(result["domain_name"]) is list:
result["domain_name"] = result["domain_name"][0]
return result
def emailsec(host):
spf = ""
dmarc = ""
result_spf = check_spf(host)
if result_spf["valid"]:
spf = result_spf["record"]
result_dmarc = check_dmarc(host)
if result_dmarc["valid"]:
dmarc = result_dmarc["record"]
return spf, dmarc

View File

@ -1,6 +0,0 @@
[metadata]
name = my_soc_site
version = 0.9
author = Joost Agterhoek
description = Flask project to learn Python and single-page websites
author_email = joostagterhoek@protonmail.com

View File

@ -1,6 +0,0 @@
#!/usr/bin/python3
from setuptools import setup
if __name__ == "__main__":
setup()

View File

@ -1,52 +0,0 @@
from base64 import decode
import json
import os
import requests
import requests_cache
from dotenv import load_dotenv
from pprint import pprint
class API_error(Exception):
pass
def environment():
requests_cache.install_cache(expire_after=360, allowable_methods=("POST"))
load_dotenv()
api_key = os.getenv("ABUSEIPDB_API")
return api_key
def lookup(api_key, host):
url = "https://api.abuseipdb.com/api/v2/check"
payload = {"ipAddress": "", "maxAgeInDays": "90"}
payload.update({"ipAddress": host})
headers = {"Accept": "application/json", "Key": api_key}
response = requests.request(
method="GET", url=url, params=payload, headers=headers, verify=False
) # TODO: remove SSL verify=False and add signed certificate if possible.
# Figure out how caching functions here: https://requests-cache.readthedocs.io/en/stable/examples.html
response_dict = json.loads(response.text)
lookup = dict.fromkeys(
["score", "last_reported", "IP_address", "CDN", "Tor", "total_reports"]
)
print(response_dict)
lookup["score"] = response_dict["data"]["abuseConfidenceScore"]
lookup["last_reported"] = response_dict["data"]["lastReportedAt"]
lookup["IP_address"] = response_dict["data"]["ipAddress"]
lookup["usage"] = response_dict["data"]["usageType"]
lookup["Tor"] = response_dict["data"]["isTor"]
lookup["total_reports"] = response_dict["data"]["totalReports"]
print(requests_cache.get_cache())
print("Cached:")
print("\n".join(requests_cache.get_cache().urls()))
return lookup
def analyse(host):
api_key = environment()
result = lookup(api_key, host)
return result

View File

@ -1,154 +0,0 @@
# TODO: make this module only have wrapper functions, no direct lookups or imports. Move all the direct lookup functions (emailsec) into (a) separate module(s).
import re
from ipaddress import ip_address
from checkdmarc.dmarc import check_dmarc
from checkdmarc.spf import check_spf
import validators
from ipwhois import IPWhois
from whois import whois
# from constants import DOMAIN, EMAIL, IPV4, IPV6, URL
from src import abuseipdb_api, virustotal_api
import socket
from urllib.parse import urlparse
import host_data
import tldextract
class Lookedup(object):
def __init__(self, host):
self.host = host
self.host_type = determine(self.host)
self = self.specific()
# TODO: consolidate all below functions if possible
def url_lookup(self):
self.domain = urlparse(self.host).netloc
self.ip_address = socket.gethostbyname(self.domain)
self.metadata = domain(self.domain)
self.email_security = spf_dmarc(self.domain)
self.vt, self.vt_dict = virustotal_api.analyse2(self.host, self.host_type)
self.abuseipdb = abuseipdb_api.analyse(self.ip_address)
return self
def ip_lookup(self):
self.metadata = domain(self.host)
self.domain = self.metadata["domain_name"]
self.email_security = spf_dmarc(self.domain)
self.vt, self.vt_dict = virustotal_api.analyse2(self.host, self.host_type)
self.abuseipdb = abuseipdb_api.analyse(self.host)
return self
def domain_lookup(self):
self.ip_address = socket.gethostbyname(self.host)
self.metadata = domain(self.host)
self.domain = self.metadata["domain_name"]
self.email_security = spf_dmarc(self.domain)
self.vt, self.vt_dict = virustotal_api.analyse2(self.host, self.host_type)
self.abuseipdb = abuseipdb_api.analyse(self.ip_address)
return self
def email_lookup(self):
self.domain = self.host.split("@")[1]
self.metadata = domain(self.domain)
self.ip_address = socket.gethostbyname(self.domain)
self.email_security = spf_dmarc(self.domain)
self.vt, self.vt_dict = virustotal_api.analyse2(self.domain, self.host_type)
self.abuseipdb = abuseipdb_api.analyse(self.ip_address)
return self
def specific(self):
if self.host_type == "url":
return self.url_lookup()
elif self.host_type == "domain":
return self.domain_lookup()
elif self.host_type == "ip":
return self.ip_lookup()
elif self.host_type == "email address":
return self.email_lookup()
def sanitize(user_input):
sanitized = []
if user_input.strip() != "":
sanitized = re.split("; |, | |\n", user_input)
return sanitized
def determine(host):
host_type = ""
if validators.url(host):
host_type = "url"
elif validators.domain(host):
host_type = "domain"
elif validators.ip_address.ipv4(host):
host_type = "ip"
elif validators.ip_address.ipv6(host):
host_type = "ip"
elif validators.email(host):
host_type = "email address"
else:
host_type = "no host"
return host_type
def extract(user_input):
hosts = []
errors = []
for item in user_input:
if validators.url(item):
hosts.append(item)
elif validators.domain(item):
hosts.append(item)
elif validators.ip_address.ipv4(item):
hosts.append(item)
elif validators.ip_address.ipv6(item):
hosts.append(item)
elif validators.email(item):
hosts.append(item)
else:
errors.append(item)
return hosts, errors
def domain(host):
result = dict(whois(host))
if type(result["creation_date"]) is list:
result["creation_date"] = result["creation_date"][0].strftime("%d-%m-%Y")
else:
result["creation_date"] = result["creation_date"].strftime("%d-%m-%Y")
if type(result["domain_name"]) is list:
result["domain_name"] = result["domain_name"][0]
return result
def spf_dmarc(domain):
spf = ""
dmarc = ""
result_spf = check_spf(domain)
if result_spf["valid"]:
spf = result_spf["record"]
result_dmarc = check_dmarc(domain)
if result_dmarc["valid"]:
dmarc = result_dmarc["record"]
return spf, dmarc
def process_input(user):
results = []
sanitized = sanitize(user)
hosts, errors = extract(sanitized)
# for host in hosts:
# host_analyzed = Host(host).lookup()
# results.append(host_analyzed)
# return results
return hosts, errors
def process_file(file_content):
hosts = []
for host in file_content:
result = Host(host)
hosts.append(result)
return hosts

View File

@ -1,111 +0,0 @@
import json
import time
import os
import requests
from dotenv import load_dotenv
from constants import URL, DOMAIN, IPV4, IPV6, domain_lookup
def environment():
load_dotenv()
api_key = os.getenv("VT_API")
return api_key
def analysis_object(api_key, host):
url = "https://www.virustotal.com/api/v3/urls"
payload = {"url": ""}
payload.update({"url": host})
headers = {
"accept": "application/json",
"content-type": "application/x-www-form-urlencoded",
"x-apikey": api_key,
}
response = requests.post(url, data=payload, headers=headers)
response_dict = json.loads(response.text)
response_id = response_dict["data"]["id"]
return response_id
def analyse_domain(api_key, host):
url = "https://www.virustotal.com/api/v3/domains/" + host
headers = {
"accept": "application/json",
"content-type": "application/x-www-form-urlencoded",
"x-apikey": api_key,
}
analysis_json = requests.get(url, headers=headers)
response_dict = json.loads(analysis_json.text)
return response_dict, analysis_json
def analyse_URL(api_key, response_id):
url = "https://www.virustotal.com/api/v3/analyses/{}".format(response_id)
headers = {"accept": "application/json", "x-apikey": api_key}
analysis_json = requests.get(url, headers=headers)
analysis_dict = json.loads(analysis_json.text)
return analysis_dict, analysis_json
# This returns a differently shaped JSON and therefore dict:
# analysis_dict keys 'data', 'meta'
# analysis_dict['data'] keys 'id', 'type', 'links', 'attributes'
# analysis_dict['data']['attributes'] keys 'stats', (numbers) 'results', (all the AV engine results) 'date', (Linux epoch timestamp) 'status'
def analyse_IP(api_key, host):
analysis_url = "https://www.virustotal.com/api/v3/ip_addresses/{}".format(host)
headers = {"accept": "application/json", "x-apikey": api_key}
analysis_json = requests.get(analysis_url, headers=headers)
response_dict = json.loads(analysis_json.text)
# Implement this: https://docs.virustotal.com/reference/ip-info
return response_dict, analysis_json
def analyse(host, host_type):
api_key = environment()
if host_type == URL:
response_id = analysis_object(api_key, host)
result, analysis_json = analyse_URL(api_key, response_id)
elif host_type == DOMAIN:
result, analysis_json = analyse_domain(api_key, host)
elif host_type == IPV4 or IPV6:
result, analysis_json = analyse_IP(api_key, host)
return result, analysis_json
def analyse2(host, host_type):
api_key = environment()
if host_type == "url":
response_id = analysis_object(api_key, host)
result, analysis_json = analyse_URL(api_key, response_id)
elif host_type == "domain" or host_type == "email address":
result, analysis_json = analyse_domain(api_key, host)
elif host_type == "ip":
result, analysis_json = analyse_IP(api_key, host)
if host_type == "url":
vt_stats = result["data"]["attributes"]["stats"]
vt_results = result["data"]["attributes"]["results"]
last_update = result["data"]["attributes"]["date"]
elif host_type == "domain" or host_type == "email address" or host_type == "ip":
vt_stats = result["data"]["attributes"]["last_analysis_stats"]
vt_results = result["data"]["attributes"]["last_analysis_results"]
last_update = result["data"]["attributes"]["last_analysis_date"]
summary = dict.fromkeys(["total", "score", "vendors", "last_update"])
total = 0
vendors = []
for key, value in vt_stats.items():
total += value
for key, value in vt_results.items():
if value["category"] == "malicious":
vendors.append(key)
summary["total"] = total
summary["score"] = vt_stats["malicious"]
summary["vendors"] = vendors
summary["last_update"] = time.strftime(
"%d-%m-%Y",
time.gmtime(last_update),
)
return summary, analysis_json

View File

@ -1,12 +0,0 @@
$(function() {
$("td[colspan=7]").find("p").hide();
$("table").click(function(event) {
event.stopPropagation();
var $target = $(event.target);
if ( $target.closest("td").attr("colspan") > 1 ) {
$target.slideUp();
} else {
$target.closest("tr").next().find("p").slideToggle();
}
});
});

View File

@ -1,179 +0,0 @@
@import url('https://fonts.googleapis.com/css2?family=DM+Mono:wght@300;400;500&display=swap');
body {
min-height: 100vh;
padding: 0;
margin: 0;
background-color: #17141d;
color: white;
font-family: 'DM Mono', monospace;
}
.headings {
color: #fdc105;
display: flex;
justify-content: center;
}
.input_container {
padding: 30px 30px;
display: flex;
justify-content: center;
}
.forms_container {
display: flex;
justify-content: around;
align-items: center;
flex-direction: column;
padding: 75px 250px;
color: #fff;
background-image: url(/static/background/cool-background-trianglify.png);
/* background: linear-gradient(to bottom right, red, yellow); */
border: 0;
transition: 0.2s;
overflow: hidden;
}
.input_form {
padding: 5px 5px;
display: flex;
}
.upload_form {
padding: 5px 5px;
display: flex;
}
.btn-warning {
position: relative;
padding: 11px 16px;
font-size: 15px;
line-height: 1.5;
border-radius: 3px;
color: #000000;
background-color: #ffc100;
border: 0;
transition: 0.2s;
opacity: 50;
overflow: hidden;
}
.btn-warning input[type="file"] {
color: transparent;
cursor: pointer;
position: absolute;
left: 0%;
top: 0%;
transform: scale(3);
opacity: 50;
transition: 0.2s;
}
.btn-warning:hover {
background-color: #eed519;
}
.submit-button {
position: relative;
padding: 11px 16px;
font-size: 15px;
line-height: 1.5;
border-radius: 3px;
color: #000000;
background-color: #ffc100;
border: 0;
transition: 0.2s;
opacity: 50;
overflow: hidden;
}
.submit-button:hover {
background-color: #eed519;
}
.table-container {
padding: 50px 50px;
border-radius: 3px;
}
.errors-container {
display: flex;
justify-content: center;
padding: 10px 10px;
border-radius: 3px;
}
.results-table {
padding: 50px 50px;
border-radius: 3px;
}
#myTable {
background-color: #fad253;
border-collapse: collapse;
width: 100%;
border: 1px solid #ddd;
font-size: 14px;
}
#myTable th {
background-color: #fad253;
text-align: left; /* Left-align text */
padding: 12px; /* Add padding */
}
#myTable tr {
/* Add a bottom border to all table rows */
border-bottom: 1px solid #ddd;
}
/*#myTable tr.header,*/ #myTable tr:hover {
/* Add a grey background color to the table header and on hover */
background-color: #ec974f;
}
#myTable td[colspan] {
padding-top: 0; padding-bottom: 0;
}
#myTable td[colspan] p {
padding-top: 12px; padding-bottom: 12px;
}
/* TEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEST */
#data {
background-color: #fad253;
border-collapse: collapse;
width: 100%;
border: 1px solid #ddd;
font-size: 14px;
}
#data th {
background-color: #fad253;
text-align: left; /* Left-align text */
padding: 12px; /* Add padding */
}
#data tr {
/* Add a bottom border to all table rows */
border-bottom: 1px solid #ddd;
}
/*#myTable tr.header,*/ #data tr:hover {
/* Add a grey background color to the table header and on hover */
background-color: #ec974f;
}
#data td[colspan] {
padding-top: 0; padding-bottom: 0;
}
#data td[colspan] p {
padding-top: 12px; padding-bottom: 12px;
}

View File

@ -1,3 +0,0 @@
h1 {
color: red;
}

View File

@ -1,12 +0,0 @@
{% extends "layout.html" %}
<!-- TODO let's see if this works -->
{% block content %}
{% include "forms.html" %}
{% include "empty_form.html" %}
{% endblock %}

View File

@ -1,98 +0,0 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
<script src="static/javascript.js"></script>
<div class="errors-container">
{% if error in errors %}
<p>
<i>
You may have mistyped: </i><b>{{errors|join(',')}}</b><i>. Try again!</i>
</i>
{% endif %}
</div>
<div class="table-container">
<div class="results-table-new">
<table id="data" class="table table-striped">
<thead>
<th>IP address</th>
<th>VirusTotal</th>
<th>AbuseIPDB</th>
<th>Domain</th>
<th>Registration country</th>
<th>Registration date</th>
<th>Email security</th>
</thead>
<tbody>
{% for host in hosts %}
<tr>
<td>{{host.host}}</td>
<td>Score: {{host.vt['score']}} / {{host.vt['total']}}<br>
Last updated: {{host.vt['last_update']}}<br>
</td>
<td>Score: {{host.abuseipdb['score']}}
Last reported: {{host.abuseipdb['last_reported']}}</td>
<td>{{host.domain}}</td>
<td>{{host.metadata['registrar_country']}}</td>
<td>{{host.metadata['creation_date']}}</td>
<td>SPF: {{host.email_security[0]}}, DMARC: {{host.email_security[1]}}</td>
</tr>
<tr>
<td colspan="7" id="info">
<p>
<b>VirusTotal</b>
<br>
{% if host.vt['score'] > 0 %}
Vendor marked this host as malicious or suspicious:
{{host.vt['vendors']|join(', ')}}
{{host.vt['score']}}
{% else %}
No vendors marked this host as malicious or suspicious.
{% endif %}
<br>
<b>AbuseIPDB</b>
<br>
IP address is: {{host.abuseipdb['IP_address']}}<br>
Is this Tor: {{host.abuseipdb['Tor']}}<br>
Usage: {{host.abuseipdb['usage']}}<br>
<br>
<!--Display some of the metadata info-->
</p>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{% block scripts %}
<script>
$(document).ready(function () {
$('#data').DataTable({
paging: true,
searching: true,
info: true,
columnDefs: [{
targets: '_all',
createdCell: function (td, cellData, rowData, row, col) {
if (cellData == 'outlook.com' && col == 0) {
$(td).css({'background-color': 'red', 'color': 'white'});
console.log(cellData);
}
if (cellData.startsWith('Score: 0') && col == 1) {
console.log("TEST", cellData);
$(td).css({'background-color': 'green', 'color': 'white'});
}
else if (!cellData.startsWith('Score: 0') && col == 1) {
$(td).css({'background-color': 'red', 'color': 'white'});
}
}
}]
});
});
</script>
{% endblock %}

View File

@ -1,87 +0,0 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
<script src="static/javascript.js"></script>
<!-- took this from https://stackoverflow.com/questions/71301182/creating-a-html-table-with-expandable-collapsible-sections, need to place the Javascript elsewhere -->
<div class="table-container">
<div class="results-table">
<table id="myTable" class="table table-striped">
<tr class="header">
<th>IP address</th>
<th>VirusTotal</th>
<th>AbuseIPDB</th>
<th>Domain</th>
<th>Registration country</th>
<th>Registration date</th>
<th>Email security</th>
</tr>
<tbody>
{% for host in hosts %}
<tr>
<td>{{host.host}}</td>
<td>Score: {{host.vt['vendors_malicious']}} / {{host.vt['vendors_total']}}<br>
Last updated: {{host.vt['last_update']}}</td>
<td>Score: {{host.abuseipdb['data']['abuseConfidenceScore']}}
Last reported: {{host.abuseipdb['data']['lastReportedAt']}}</td>
<td>{{host.domain}}</td>
<td>{{host.metadata['registrar_country']}}</td>
<td>{{host.metadata['creation_date']}}</td>
<td>SPF: {{host.emailsec[0]}}, DMARC: {{host.emailsec[1]}}</td>
</tr>
<tr>
<td colspan="7" id="info">
<p>
Vendor marked this host as malicious or suspicious:
{% for key, value in host.vt['all_vendors'].items() if value['category'] in ['malicious', 'suspicious'] %}
{{key}}
{% endfor %}
</p>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{# {% include "test-table.html" %} #}
<!-- important to know: uncommenting the line above makes the red/green JavaScript not work anymore, BUT makes the collapsible table row table work! Go figure it out XD -->
{% block scripts %}
{#{% for host in hosts %}#}
{#whoopsie = JSON.parse({{ host.vt_json.text | tojson }}) {% endfor %}#}
{#{{whoopsie}}#}
<script>
$(document).ready(function () {
$('myTable').DataTable({
paging: false,
searching: false,
info: false,
columnDefs: [{
targets: '_all',
createdCell: function (td, cellData, rowData, row, col) {
if (cellData == 'outlook.com' && col == 0) {
$(td).css({'background-color': 'red', 'color': 'white'});
}
if (cellData.startsWith('Score: 0' && col == 1)) {
$(td).css({'background-color': 'orange', 'color': 'white'});
}
if (cellData > 0 && col == 2) {
$(td).css({'background-color': 'grey', 'color': 'white'});
}
if (cellData > 0 && col == 3) {
$(td).css({'background-color': 'green', 'color': 'white'});
}
},
}
]
})
});
</script>
{% endblock %}

View File

@ -1,17 +0,0 @@
{% extends "layout.html" %}
{% block content %}
{# For multiple results, maybe make a top results overview and then a navigation menu to specific URL pages? See: https://realpython.com/primer-on-jinja-templating/#include-a-navigation-menu #}
{% include "forms.html" %}
{# {% include "results-overview.html" %} #}
{% include "results-overview-new.html" %}
{% include "results-detail.html" %}
{% include "empty_form.html" %}
{% endblock %}

View File

@ -1,22 +0,0 @@
import csv
from io import StringIO
from validators import ipv4, ipv6, url, domain
def extract(uploaded):
hosts = []
content = uploaded.read()
decoded = content.decode("utf-8")
file = StringIO(decoded)
csv_data = csv.reader(file, delimiter=",")
for row in csv_data:
for value in row:
if url(value):
hosts.append(value)
elif domain(value):
hosts.append(value)
elif ipv4(value):
hosts.append(value)
elif ipv6(value):
hosts.append(value)
return hosts